mirror of
https://github.com/MervinPraison/PraisonAI.git
synced 2026-11-09 03:39:37 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Mervin Praison
|
5a13571fdd | ||
|
Mervin Praison
|
e3b48c2526 | ||
|
Mervin Praison
|
8d32238b4c | ||
|
Mervin Praison
|
c17b1298bb | ||
|
Mervin Praison
|
ec6c150315 | ||
|
Mervin Praison
|
3d26cedd5d | ||
|
Mervin Praison
|
8c735a60b2 | ||
|
Mervin Praison
|
2f5f2f1c13 | ||
|
Mervin Praison
|
0253073c68 |
No files matched your search
@@ -171,12 +171,13 @@ npm install praisonai
|
||||
|
||||
| Feature | Code | Docs |
|
||||
|---------|:----:|:----:|
|
||||
| 🚀 Single Agent | [Example](examples/python/agents/single-agent.py) | [📖](https://docs.praison.ai/agents/single) |
|
||||
| 🤝 Multi Agents | [Example](examples/python/general/mini_agents_example.py) | [📖](https://docs.praison.ai/concepts/agents) |
|
||||
| 🤖 Auto Agents | [Example](examples/python/general/auto_agents_example.py) | [📖](https://docs.praison.ai/features/autoagents) |
|
||||
| 🔄 Self Reflection AI Agents | [Example](examples/python/concepts/self-reflection-details.py) | [📖](https://docs.praison.ai/features/selfreflection) |
|
||||
| 🧠 Reasoning AI Agents | [Example](examples/python/concepts/reasoning-extraction.py) | [📖](https://docs.praison.ai/features/reasoning) |
|
||||
| 👁️ Multi Modal AI Agents | [Example](examples/python/general/multimodal.py) | [📖](https://docs.praison.ai/features/multimodal) |
|
||||
| **🤖 Core Agents** | | |
|
||||
| ↳ Single Agent | [Example](examples/python/agents/single-agent.py) | [📖](https://docs.praison.ai/agents/single) |
|
||||
| ↳ Multi Agents | [Example](examples/python/general/mini_agents_example.py) | [📖](https://docs.praison.ai/concepts/agents) |
|
||||
| ↳ Auto Agents | [Example](examples/python/general/auto_agents_example.py) | [📖](https://docs.praison.ai/features/autoagents) |
|
||||
| ↳ Self Reflection AI Agents | [Example](examples/python/concepts/self-reflection-details.py) | [📖](https://docs.praison.ai/features/selfreflection) |
|
||||
| ↳ Reasoning AI Agents | [Example](examples/python/concepts/reasoning-extraction.py) | [📖](https://docs.praison.ai/features/reasoning) |
|
||||
| ↳ Multi Modal AI Agents | [Example](examples/python/general/multimodal.py) | [📖](https://docs.praison.ai/features/multimodal) |
|
||||
| **🔄 Workflows** | | |
|
||||
| ↳ Simple Workflow | [Example](examples/python/workflows/simple_workflow.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow with Agents | [Example](examples/python/workflows/workflow_with_agents.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
@@ -188,10 +189,7 @@ npm install praisonai
|
||||
| ↳ Workflow Branching | [Example](examples/python/workflows/workflow_branching.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow Early Stop | [Example](examples/python/workflows/workflow_early_stop.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow Checkpoints | [Example](examples/python/workflows/workflow_checkpoints.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| 📚 Add Custom Knowledge | [Example](examples/python/concepts/knowledge-agents.py) | [📖](https://docs.praison.ai/features/knowledge) |
|
||||
| 🧠 Memory (Short & Long Term) | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| 📄 Chat with PDF Agents | [Example](examples/python/concepts/chat-with-pdf.py) | [📖](https://docs.praison.ai/features/chat-with-pdf) |
|
||||
| **💻 Code** | | |
|
||||
| **💻 Code & Development** | | |
|
||||
| ↳ Code Interpreter Agents | [Example](examples/python/agents/code-agent.py) | [📖](https://docs.praison.ai/features/codeagent) |
|
||||
| ↳ AI Code Editing Tools | [Example](examples/python/code/code_editing_example.py) | [📖](https://docs.praison.ai/code/editing) |
|
||||
| ↳ External Agents (All) | [Example](examples/python/code/external_agents_example.py) | [📖](https://docs.praison.ai/code/external-agents) |
|
||||
@@ -199,65 +197,78 @@ npm install praisonai
|
||||
| ↳ Gemini CLI | [Example](examples/python/code/gemini_cli_example.py) | [📖](https://docs.praison.ai/code/gemini-cli) |
|
||||
| ↳ Codex CLI | [Example](examples/python/code/codex_cli_example.py) | [📖](https://docs.praison.ai/code/codex-cli) |
|
||||
| ↳ Cursor CLI | [Example](examples/python/code/cursor_cli_example.py) | [📖](https://docs.praison.ai/code/cursor-cli) |
|
||||
| 📚 RAG Agents | [Example](examples/python/concepts/rag-agents.py) | [📖](https://docs.praison.ai/features/rag) |
|
||||
| 🤔 Async & Parallel Processing | [Example](examples/python/general/async_example.py) | [📖](https://docs.praison.ai/features/async) |
|
||||
| 🔢 Math Agents | [Example](examples/python/agents/math-agent.py) | [📖](https://docs.praison.ai/features/mathagent) |
|
||||
| 🎯 Structured Output Agents | [Example](examples/python/general/structured_agents_example.py) | [📖](https://docs.praison.ai/features/structured) |
|
||||
| 🔗 LangChain Integrated Agents | [Example](examples/python/general/langchain_example.py) | [📖](https://docs.praison.ai/features/langchain) |
|
||||
| 📞 Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
|
||||
| 🛠️ 100+ Custom Tools | [Example](examples/python/general/tools_example.py) | [📖](https://docs.praison.ai/tools/tools) |
|
||||
| 📄 YAML Configuration | [Example](examples/cookbooks/yaml/secondary_market_research_agents.yaml) | [📖](https://docs.praison.ai/developers/agents-playbook) |
|
||||
| 💯 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
|
||||
| 🔬 Deep Research Agents | [Example](examples/python/agents/research-agent.py) | [📖](https://docs.praison.ai/agents/deep-research) |
|
||||
| 🔄 Query Rewriter Agent | [Example](#5-query-rewriter-agent) | [📖](https://docs.praison.ai/agents/query-rewriter) |
|
||||
| 🌐 Native Web Search | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/agents/websearch) |
|
||||
| 📥 Web Fetch (Anthropic) | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| 💾 Prompt Caching | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| 🧠 Claude Memory Tool | [Example](#claude-memory-tool-cli) | [📖](https://docs.praison.ai/features/claude-memory-tool) |
|
||||
| 💾 File-Based Memory | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| 🔍 Built-in Search Tools | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/tools/tavily) |
|
||||
| 🔎 Unified Web Search | [Example](src/praisonai-agents/examples/web_search_example.py) | [📖](https://docs.praison.ai/tools/web-search) |
|
||||
| 📋 Planning Mode | [Example](examples/python/agents/planning-agent.py) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🔧 Planning Tools | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🧠 Planning Reasoning | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🔌 MCP Transports | [Example](examples/python/mcp/mcp-transports-overview.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| 🌐 WebSocket MCP | [Example](examples/python/mcp/websocket-mcp.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| 🔐 MCP Security | [Example](examples/python/mcp/mcp-security.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| 🔄 MCP Resumability | [Example](examples/python/mcp/mcp-resumability.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ⚡ Fast Context | [Example](examples/context/00_agent_fast_context_basic.py) | [📖](https://docs.praison.ai/features/fast-context) |
|
||||
| 🖼️ Image Generation Agent | [Example](examples/python/image/image-agent.py) | [📖](https://docs.praison.ai/features/image-generation) |
|
||||
| 📷 Image to Text Agent | [Example](examples/python/agents/image-to-text-agent.py) | [📖](https://docs.praison.ai/agents/image-to-text) |
|
||||
| 🎬 Video Agent | [Example](examples/python/agents/video-agent.py) | [📖](https://docs.praison.ai/agents/video) |
|
||||
| 📊 Data Analyst Agent | [Example](examples/python/agents/data-analyst-agent.py) | [📖](https://docs.praison.ai/agents/data-analyst) |
|
||||
| 💰 Finance Agent | [Example](examples/python/agents/finance-agent.py) | [📖](https://docs.praison.ai/agents/finance) |
|
||||
| 🛒 Shopping Agent | [Example](examples/python/agents/shopping-agent.py) | [📖](https://docs.praison.ai/agents/shopping) |
|
||||
| ⭐ Recommendation Agent | [Example](examples/python/agents/recommendation-agent.py) | [📖](https://docs.praison.ai/agents/recommendation) |
|
||||
| 📖 Wikipedia Agent | [Example](examples/python/agents/wikipedia-agent.py) | [📖](https://docs.praison.ai/agents/wikipedia) |
|
||||
| 💻 Programming Agent | [Example](examples/python/agents/programming-agent.py) | [📖](https://docs.praison.ai/agents/programming) |
|
||||
| 📝 Markdown Agent | [Example](examples/python/agents/markdown-agent.py) | [📖](https://docs.praison.ai/agents/markdown) |
|
||||
| 📝 Prompt Expander Agent | [Example](#prompt-expansion) | [📖](https://docs.praison.ai/agents/prompt-expander) |
|
||||
| 🔀 Model Router | [Example](examples/python/agents/router-agent-cost-optimization.py) | [📖](https://docs.praison.ai/features/model-router) |
|
||||
| ⛓️ Prompt Chaining | [Example](examples/python/general/prompt_chaining.py) | [📖](https://docs.praison.ai/features/promptchaining) |
|
||||
| 🔍 Evaluator Optimiser | [Example](examples/python/general/evaluator-optimiser.py) | [📖](https://docs.praison.ai/features/evaluator-optimiser) |
|
||||
| 👷 Orchestrator Workers | [Example](examples/python/general/orchestrator-workers.py) | [📖](https://docs.praison.ai/features/orchestrator-worker) |
|
||||
| ⚡ Parallelisation | [Example](examples/python/general/parallelisation.py) | [📖](https://docs.praison.ai/features/parallelisation) |
|
||||
| 🔁 Repetitive Agents | [Example](examples/python/concepts/repetitive-agents.py) | [📖](https://docs.praison.ai/features/repetitive) |
|
||||
| 🤝 Agent Handoffs | [Example](examples/python/handoff/handoff_basic.py) | [📖](https://docs.praison.ai/features/handoffs) |
|
||||
| 🛡️ Guardrails | [Example](examples/python/guardrails/comprehensive-guardrails-example.py) | [📖](https://docs.praison.ai/features/guardrails) |
|
||||
| 💬 Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
|
||||
| ✅ Human Approval | [Example](examples/python/general/human_approval_example.py) | [📖](https://docs.praison.ai/features/approval) |
|
||||
| 🔄 Stateful Agents | [Example](examples/python/stateful/workflow-state-example.py) | [📖](https://docs.praison.ai/features/stateful-agents) |
|
||||
| 🤖 Autonomous Workflow | [Example](examples/python/general/autonomous-agent.py) | [📖](https://docs.praison.ai/features/autonomous-workflow) |
|
||||
| 📜 Rules & Instructions | [Example](#6-rules--instructions) | [📖](https://docs.praison.ai/features/rules) |
|
||||
| 🪝 Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
|
||||
| 📈 Telemetry | [Example](examples/python/telemetry/production-telemetry-example.py) | [📖](https://docs.praison.ai/features/telemetry) |
|
||||
| 📹 Camera Integration | [Example](examples/python/camera/) | [📖](https://docs.praison.ai/features/camera-integration) |
|
||||
| 📄 Project Docs (.praison/docs/) | [Example](#docs-cli) | [📖](https://docs.praison.ai/docs/cli/docs) |
|
||||
| 🔌 MCP Config Management | [Example](#mcp-config-cli) | [📖](https://docs.praison.ai/docs/cli/mcp) |
|
||||
| 💬 AI Commit Messages | [Example](#ai-commit-cli) | [📖](https://docs.praison.ai/docs/cli/commit) |
|
||||
| @ @Mentions in Prompts | [Example](#mentions-in-prompts) | [📖](https://docs.praison.ai/docs/cli/mentions) |
|
||||
| 💾 Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| 📜 History in Context | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| **🧠 Memory & Knowledge** | | |
|
||||
| ↳ Memory (Short & Long Term) | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| ↳ File-Based Memory | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| ↳ Claude Memory Tool | [Example](#claude-memory-tool-cli) | [📖](https://docs.praison.ai/features/claude-memory-tool) |
|
||||
| ↳ Add Custom Knowledge | [Example](examples/python/concepts/knowledge-agents.py) | [📖](https://docs.praison.ai/features/knowledge) |
|
||||
| ↳ RAG Agents | [Example](examples/python/concepts/rag-agents.py) | [📖](https://docs.praison.ai/features/rag) |
|
||||
| ↳ Chat with PDF Agents | [Example](examples/python/concepts/chat-with-pdf.py) | [📖](https://docs.praison.ai/features/chat-with-pdf) |
|
||||
| **🔬 Research & Intelligence** | | |
|
||||
| ↳ Deep Research Agents | [Example](examples/python/agents/research-agent.py) | [📖](https://docs.praison.ai/agents/deep-research) |
|
||||
| ↳ Query Rewriter Agent | [Example](#5-query-rewriter-agent) | [📖](https://docs.praison.ai/agents/query-rewriter) |
|
||||
| ↳ Native Web Search | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/agents/websearch) |
|
||||
| ↳ Built-in Search Tools | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/tools/tavily) |
|
||||
| ↳ Unified Web Search | [Example](src/praisonai-agents/examples/web_search_example.py) | [📖](https://docs.praison.ai/tools/web-search) |
|
||||
| ↳ Web Fetch (Anthropic) | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| **📋 Planning & Execution** | | |
|
||||
| ↳ Planning Mode | [Example](examples/python/agents/planning-agent.py) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Planning Tools | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Planning Reasoning | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Prompt Chaining | [Example](examples/python/general/prompt_chaining.py) | [📖](https://docs.praison.ai/features/promptchaining) |
|
||||
| ↳ Evaluator Optimiser | [Example](examples/python/general/evaluator-optimiser.py) | [📖](https://docs.praison.ai/features/evaluator-optimiser) |
|
||||
| ↳ Orchestrator Workers | [Example](examples/python/general/orchestrator-workers.py) | [📖](https://docs.praison.ai/features/orchestrator-worker) |
|
||||
| **👥 Specialized Agents** | | |
|
||||
| ↳ Data Analyst Agent | [Example](examples/python/agents/data-analyst-agent.py) | [📖](https://docs.praison.ai/agents/data-analyst) |
|
||||
| ↳ Finance Agent | [Example](examples/python/agents/finance-agent.py) | [📖](https://docs.praison.ai/agents/finance) |
|
||||
| ↳ Shopping Agent | [Example](examples/python/agents/shopping-agent.py) | [📖](https://docs.praison.ai/agents/shopping) |
|
||||
| ↳ Recommendation Agent | [Example](examples/python/agents/recommendation-agent.py) | [📖](https://docs.praison.ai/agents/recommendation) |
|
||||
| ↳ Wikipedia Agent | [Example](examples/python/agents/wikipedia-agent.py) | [📖](https://docs.praison.ai/agents/wikipedia) |
|
||||
| ↳ Programming Agent | [Example](examples/python/agents/programming-agent.py) | [📖](https://docs.praison.ai/agents/programming) |
|
||||
| ↳ Math Agents | [Example](examples/python/agents/math-agent.py) | [📖](https://docs.praison.ai/features/mathagent) |
|
||||
| ↳ Markdown Agent | [Example](examples/python/agents/markdown-agent.py) | [📖](https://docs.praison.ai/agents/markdown) |
|
||||
| ↳ Prompt Expander Agent | [Example](#prompt-expansion) | [📖](https://docs.praison.ai/agents/prompt-expander) |
|
||||
| **🎨 Media & Multimodal** | | |
|
||||
| ↳ Image Generation Agent | [Example](examples/python/image/image-agent.py) | [📖](https://docs.praison.ai/features/image-generation) |
|
||||
| ↳ Image to Text Agent | [Example](examples/python/agents/image-to-text-agent.py) | [📖](https://docs.praison.ai/agents/image-to-text) |
|
||||
| ↳ Video Agent | [Example](examples/python/agents/video-agent.py) | [📖](https://docs.praison.ai/agents/video) |
|
||||
| ↳ Camera Integration | [Example](examples/python/camera/) | [📖](https://docs.praison.ai/features/camera-integration) |
|
||||
| **🔌 Protocols & Integration** | | |
|
||||
| ↳ MCP Transports | [Example](examples/python/mcp/mcp-transports-overview.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| ↳ WebSocket MCP | [Example](examples/python/mcp/websocket-mcp.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ↳ MCP Security | [Example](examples/python/mcp/mcp-security.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| ↳ MCP Resumability | [Example](examples/python/mcp/mcp-resumability.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ↳ MCP Config Management | [Example](#mcp-config-cli) | [📖](https://docs.praison.ai/docs/cli/mcp) |
|
||||
| ↳ LangChain Integrated Agents | [Example](examples/python/general/langchain_example.py) | [📖](https://docs.praison.ai/features/langchain) |
|
||||
| **🛡️ Safety & Control** | | |
|
||||
| ↳ Guardrails | [Example](examples/python/guardrails/comprehensive-guardrails-example.py) | [📖](https://docs.praison.ai/features/guardrails) |
|
||||
| ↳ Human Approval | [Example](examples/python/general/human_approval_example.py) | [📖](https://docs.praison.ai/features/approval) |
|
||||
| ↳ Rules & Instructions | [Example](#6-rules--instructions) | [📖](https://docs.praison.ai/features/rules) |
|
||||
| **⚙️ Advanced Features** | | |
|
||||
| ↳ Async & Parallel Processing | [Example](examples/python/general/async_example.py) | [📖](https://docs.praison.ai/features/async) |
|
||||
| ↳ Parallelisation | [Example](examples/python/general/parallelisation.py) | [📖](https://docs.praison.ai/features/parallelisation) |
|
||||
| ↳ Repetitive Agents | [Example](examples/python/concepts/repetitive-agents.py) | [📖](https://docs.praison.ai/features/repetitive) |
|
||||
| ↳ Agent Handoffs | [Example](examples/python/handoff/handoff_basic.py) | [📖](https://docs.praison.ai/features/handoffs) |
|
||||
| ↳ Stateful Agents | [Example](examples/python/stateful/workflow-state-example.py) | [📖](https://docs.praison.ai/features/stateful-agents) |
|
||||
| ↳ Autonomous Workflow | [Example](examples/python/general/autonomous-agent.py) | [📖](https://docs.praison.ai/features/autonomous-workflow) |
|
||||
| ↳ Structured Output Agents | [Example](examples/python/general/structured_agents_example.py) | [📖](https://docs.praison.ai/features/structured) |
|
||||
| ↳ Model Router | [Example](examples/python/agents/router-agent-cost-optimization.py) | [📖](https://docs.praison.ai/features/model-router) |
|
||||
| ↳ Prompt Caching | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| ↳ Fast Context | [Example](examples/context/00_agent_fast_context_basic.py) | [📖](https://docs.praison.ai/features/fast-context) |
|
||||
| **🛠️ Tools & Configuration** | | |
|
||||
| ↳ 100+ Custom Tools | [Example](examples/python/general/tools_example.py) | [📖](https://docs.praison.ai/tools/tools) |
|
||||
| ↳ YAML Configuration | [Example](examples/cookbooks/yaml/secondary_market_research_agents.yaml) | [📖](https://docs.praison.ai/developers/agents-playbook) |
|
||||
| ↳ 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
|
||||
| ↳ Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
|
||||
| ↳ Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
|
||||
| **📊 Monitoring & Management** | | |
|
||||
| ↳ Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
|
||||
| ↳ Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| ↳ History in Context | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| ↳ Telemetry | [Example](examples/python/telemetry/production-telemetry-example.py) | [📖](https://docs.praison.ai/features/telemetry) |
|
||||
| ↳ Project Docs (.praison/docs/) | [Example](#docs-cli) | [📖](https://docs.praison.ai/docs/cli/docs) |
|
||||
| ↳ AI Commit Messages | [Example](#ai-commit-cli) | [📖](https://docs.praison.ai/docs/cli/commit) |
|
||||
| ↳ @Mentions in Prompts | [Example](#mentions-in-prompts) | [📖](https://docs.praison.ai/docs/cli/mentions) |
|
||||
| **🖥️ CLI Features** | | |
|
||||
| ↳ Slash Commands | [Example](examples/python/cli/slash_commands_example.py) | [📖](https://docs.praison.ai/docs/cli/slash-commands) |
|
||||
| ↳ Autonomy Modes | [Example](examples/python/cli/autonomy_modes_example.py) | [📖](https://docs.praison.ai/docs/cli/autonomy-modes) |
|
||||
@@ -266,6 +277,15 @@ npm install praisonai
|
||||
| ↳ Interactive TUI | [Example](examples/python/cli/interactive_tui_example.py) | [📖](https://docs.praison.ai/docs/cli/interactive-tui) |
|
||||
| ↳ Git Integration | [Example](examples/python/cli/git_integration_example.py) | [📖](https://docs.praison.ai/docs/cli/git-integration) |
|
||||
| ↳ Sandbox Execution | [Example](examples/python/cli/sandbox_execution_example.py) | [📖](https://docs.praison.ai/docs/cli/sandbox-execution) |
|
||||
| ↳ CLI Compare | [Example](examples/compare/cli_compare_basic.py) | [📖](https://docs.praison.ai/docs/cli/compare) |
|
||||
| **🧪 Evaluation** | | |
|
||||
| ↳ Accuracy Evaluation | [Example](examples/eval/accuracy_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Performance Evaluation | [Example](examples/eval/performance_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Reliability Evaluation | [Example](examples/eval/reliability_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Criteria Evaluation | [Example](examples/eval/criteria_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| **🎯 Agent Skills** | | |
|
||||
| ↳ Skills Management | [Example](examples/skills/basic_skill_usage.py) | [📖](https://docs.praison.ai/features/skills) |
|
||||
| ↳ Custom Skills | [Example](examples/skills/custom_skill_example.py) | [📖](https://docs.praison.ai/features/skills) |
|
||||
| **⏰ 24/7 Scheduling** | | |
|
||||
| ↳ Agent Scheduler | [Example](examples/python/scheduled_agents/news_checker_live.py) | [📖](https://docs.praison.ai/docs/cli/scheduler) |
|
||||
|
||||
@@ -1246,62 +1266,19 @@ praisonai "Complex task" --metrics --planning
|
||||
|
||||
### Scheduler CLI:
|
||||
|
||||
Run agents 24/7 with built-in safety features. Two modes available:
|
||||
|
||||
**Direct Prompt Mode (Simple):**
|
||||
```bash
|
||||
# No YAML needed - just provide your task
|
||||
praisonai schedule "Check AI news and summarize" --interval hourly
|
||||
|
||||
# With options
|
||||
praisonai schedule "Monitor server logs" \
|
||||
--interval "*/30m" \
|
||||
--timeout 120 \
|
||||
--max-cost 2.00 \
|
||||
--verbose
|
||||
praisonai schedule start <name> "task" --interval hourly
|
||||
praisonai schedule list
|
||||
praisonai schedule logs <name> [--follow]
|
||||
praisonai schedule stop <name>
|
||||
praisonai schedule restart <name>
|
||||
praisonai schedule delete <name>
|
||||
praisonai schedule describe <name>
|
||||
praisonai schedule save <name> [file.yaml]
|
||||
praisonai schedule "task" --interval hourly # foreground mode
|
||||
praisonai schedule agents.yaml # foreground mode
|
||||
```
|
||||
|
||||
**YAML Mode (Advanced):**
|
||||
```bash
|
||||
# Use agents.yaml for complex configurations
|
||||
praisonai schedule agents.yaml
|
||||
|
||||
# With overrides
|
||||
praisonai schedule agents.yaml \
|
||||
--interval "*/15m" \
|
||||
--timeout 60 \
|
||||
--max-cost 1.00
|
||||
|
||||
# Stop: Press Ctrl+C
|
||||
```
|
||||
|
||||
**agents.yaml with schedule:**
|
||||
```yaml
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "AI News Monitor"
|
||||
instructions: "Search and summarize AI news"
|
||||
tools:
|
||||
- search_tool
|
||||
|
||||
task: "Search for latest AI news"
|
||||
|
||||
schedule:
|
||||
interval: "hourly" # hourly, daily, */30m, */6h
|
||||
max_retries: 3 # Retry on failure
|
||||
run_immediately: true # Run once on start
|
||||
timeout: 60 # Max 60s per execution
|
||||
max_cost: 1.00 # Stop after $1.00 spent
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- ⏱️ Timeout protection (prevent runaway executions)
|
||||
- 💰 Cost monitoring with budget limits
|
||||
- 📊 Real-time statistics (success rate, cost, runtime)
|
||||
- 🔄 Automatic retry with exponential backoff
|
||||
- 🛡️ Auto-stops when budget reached
|
||||
|
||||
### Image Processing CLI:
|
||||
```bash
|
||||
# Process images with vision-based tasks
|
||||
|
||||
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=2.3.67" \
|
||||
"praisonai>=2.3.75" \
|
||||
"praisonai[chat]" \
|
||||
"embedchain[github,youtube]"
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=2.3.67" \
|
||||
"praisonai>=2.3.75" \
|
||||
"praisonai[ui]" \
|
||||
"praisonai[chat]" \
|
||||
"praisonai[realtime]" \
|
||||
|
||||
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=2.3.67" \
|
||||
"praisonai>=2.3.75" \
|
||||
"praisonai[ui]" \
|
||||
"praisonai[crewai]"
|
||||
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
"""
|
||||
CLI Compare Example - Advanced Usage
|
||||
|
||||
This example demonstrates advanced comparison features including
|
||||
custom mode configurations and result analysis.
|
||||
"""
|
||||
|
||||
from praisonai.cli.features.compare import (
|
||||
CompareHandler,
|
||||
CompareResult,
|
||||
ModeResult,
|
||||
get_mode_config,
|
||||
save_compare_result,
|
||||
format_comparison_table,
|
||||
)
|
||||
|
||||
# Create custom mode configurations
|
||||
custom_modes = {
|
||||
"fast": {"llm": "gpt-4o-mini"},
|
||||
"accurate": {"llm": "gpt-4o", "planning": True},
|
||||
"research": {"research": True, "web_search": True},
|
||||
}
|
||||
|
||||
# Create handler with verbose output
|
||||
handler = CompareHandler(verbose=True)
|
||||
|
||||
# Example: Manually create comparison results for analysis
|
||||
# (useful for testing without API calls)
|
||||
mock_result = CompareResult(
|
||||
query="What are the latest AI trends?",
|
||||
comparisons=[
|
||||
ModeResult(
|
||||
mode="basic",
|
||||
output="AI trends include LLMs, multimodal models, and agents.",
|
||||
execution_time_ms=1234.5,
|
||||
model_used="gpt-4o-mini",
|
||||
tools_used=[]
|
||||
),
|
||||
ModeResult(
|
||||
mode="tools",
|
||||
output="Based on recent search results, AI trends in 2024 include...",
|
||||
execution_time_ms=2567.8,
|
||||
model_used="gpt-4o-mini",
|
||||
tools_used=["internet_search"]
|
||||
),
|
||||
ModeResult(
|
||||
mode="research",
|
||||
output="Comprehensive research shows that AI trends...",
|
||||
execution_time_ms=5432.1,
|
||||
model_used="gpt-4o-mini",
|
||||
tools_used=["internet_search", "web_scraper"]
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
# Get summary statistics
|
||||
summary = mock_result.get_summary()
|
||||
print("Comparison Summary:")
|
||||
print(f" Fastest mode: {summary['fastest']} ({summary['fastest_time_ms']:.1f}ms)")
|
||||
print(f" Slowest mode: {summary['slowest']} ({summary['slowest_time_ms']:.1f}ms)")
|
||||
|
||||
# Format as table
|
||||
print("\n" + format_comparison_table(mock_result))
|
||||
|
||||
# Convert to JSON for storage
|
||||
print("\nJSON Output:")
|
||||
print(mock_result.to_json())
|
||||
|
||||
# Save to file
|
||||
# save_compare_result(mock_result, "advanced_comparison.json")
|
||||
|
||||
print("\n--- CLI Usage Examples ---")
|
||||
print("""
|
||||
# Basic comparison
|
||||
praisonai "What is AI?" --compare "basic,tools"
|
||||
|
||||
# Compare with specific model
|
||||
praisonai "Explain quantum computing" --compare "basic,planning,research" --model gpt-4o
|
||||
|
||||
# Save results to file
|
||||
praisonai "Latest tech news" --compare "basic,tools,web_search" --compare-output results.json
|
||||
|
||||
# Verbose output
|
||||
praisonai "Write a poem" --compare "basic,planning" --verbose
|
||||
""")
|
||||
@@ -0,0 +1,45 @@
|
||||
"""
|
||||
CLI Compare Example - Basic Usage
|
||||
|
||||
This example demonstrates how to compare different CLI modes programmatically.
|
||||
"""
|
||||
|
||||
from praisonai.cli.features.compare import (
|
||||
CompareHandler,
|
||||
get_mode_config,
|
||||
list_available_modes,
|
||||
parse_modes,
|
||||
)
|
||||
|
||||
# List all available comparison modes
|
||||
print("Available modes:")
|
||||
for mode in list_available_modes():
|
||||
config = get_mode_config(mode)
|
||||
print(f" - {mode}: {config}")
|
||||
|
||||
# Parse modes from comma-separated string
|
||||
modes = parse_modes("basic,tools,planning")
|
||||
print(f"\nParsed modes: {modes}")
|
||||
|
||||
# Create compare handler
|
||||
handler = CompareHandler(verbose=True)
|
||||
|
||||
# Compare modes (this will run actual agent calls)
|
||||
# Uncomment to run:
|
||||
# result = handler.compare(
|
||||
# query="What is artificial intelligence?",
|
||||
# modes=["basic", "tools"],
|
||||
# model="gpt-4o-mini"
|
||||
# )
|
||||
# handler.print_result(result)
|
||||
|
||||
# Save results to file
|
||||
# result = handler.execute(
|
||||
# query="Explain machine learning",
|
||||
# modes_str="basic,planning",
|
||||
# output_path="comparison_results.json"
|
||||
# )
|
||||
|
||||
print("\nTo run comparison via CLI:")
|
||||
print(' praisonai "What is AI?" --compare "basic,tools,planning"')
|
||||
print(' praisonai "What is AI?" --compare "basic,research" --compare-output results.json')
|
||||
@@ -0,0 +1,69 @@
|
||||
"""
|
||||
Accuracy Evaluation Example
|
||||
|
||||
This example demonstrates how to evaluate agent output accuracy
|
||||
by comparing against expected outputs using LLM-as-judge.
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.eval import AccuracyEvaluator
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
if has_api_key:
|
||||
# Create a simple math tutor agent
|
||||
agent = Agent(
|
||||
instructions="You are a math tutor. Answer math questions concisely with just the number."
|
||||
)
|
||||
|
||||
# Create accuracy evaluator
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text="What is 2 + 2?",
|
||||
expected_output="4",
|
||||
num_iterations=3, # Run 3 times for statistical significance
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Run evaluation
|
||||
result = evaluator.run(print_summary=True)
|
||||
|
||||
# Check results
|
||||
print(f"\nAverage Score: {result.avg_score}/10")
|
||||
print(f"Passed: {result.passed}")
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Using mock evaluation...")
|
||||
|
||||
# Create evaluator without agent for mock testing
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=lambda x: "4", # Mock function
|
||||
input_text="What is 2 + 2?",
|
||||
expected_output="4",
|
||||
num_iterations=1,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Test with mock output
|
||||
try:
|
||||
result = evaluator.run(print_summary=True)
|
||||
print(f"\nMock Average Score: {result.avg_score}/10")
|
||||
print(f"Mock Passed: {result.passed}")
|
||||
except Exception as e:
|
||||
print(f"Mock evaluation failed: {e}")
|
||||
print("This is expected without an API key")
|
||||
|
||||
# You can also evaluate pre-generated outputs
|
||||
print("\n--- Testing pre-generated output evaluation ---")
|
||||
output = "The answer is 4"
|
||||
|
||||
if has_api_key:
|
||||
try:
|
||||
result2 = evaluator.evaluate_output(output, print_summary=True)
|
||||
print(f"Pre-generated output score: {result2.avg_score}/10")
|
||||
except Exception as e:
|
||||
print(f"Pre-generated evaluation failed: {e}")
|
||||
else:
|
||||
print("⚠️ Skipping pre-generated evaluation (no API key)")
|
||||
print("To run full evaluation, set OPENAI_API_KEY environment variable")
|
||||
@@ -0,0 +1,105 @@
|
||||
"""
|
||||
Batch Evaluation Example
|
||||
|
||||
This example demonstrates how to run batch evaluations
|
||||
from a JSON test file.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import tempfile
|
||||
from praisonaiagents.eval import AccuracyEvaluator
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
# Create a test file with multiple test cases
|
||||
test_cases = [
|
||||
{
|
||||
"input": "What is 2 + 2?",
|
||||
"expected": "4"
|
||||
},
|
||||
{
|
||||
"input": "What is the capital of France?",
|
||||
"expected": "Paris"
|
||||
},
|
||||
{
|
||||
"input": "What color is the sky?",
|
||||
"expected": "Blue"
|
||||
}
|
||||
]
|
||||
|
||||
# Save test cases to a temporary file
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
|
||||
json.dump(test_cases, f)
|
||||
test_file = f.name
|
||||
|
||||
print(f"Test file created: {test_file}")
|
||||
print(f"Running {len(test_cases)} test cases...")
|
||||
|
||||
if has_api_key:
|
||||
# You can use the CLI for batch evaluation:
|
||||
# praisonai eval batch --agent agents.yaml --test-file tests.json --batch-type accuracy
|
||||
|
||||
# Or programmatically:
|
||||
from praisonaiagents import Agent
|
||||
|
||||
agent = Agent(
|
||||
instructions="You are a helpful assistant. Answer questions concisely."
|
||||
)
|
||||
|
||||
results = []
|
||||
for i, test_case in enumerate(test_cases):
|
||||
print(f"\nTest {i + 1}: {test_case['input']}")
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text=test_case["input"],
|
||||
expected_output=test_case["expected"],
|
||||
num_iterations=1
|
||||
)
|
||||
|
||||
result = evaluator.run()
|
||||
results.append({
|
||||
"input": test_case["input"],
|
||||
"expected": test_case["expected"],
|
||||
"score": result.avg_score,
|
||||
"passed": result.passed
|
||||
})
|
||||
|
||||
print(f" Score: {result.avg_score}/10")
|
||||
print(f" Passed: {result.passed}")
|
||||
|
||||
# Summary
|
||||
passed = sum(1 for r in results if r["passed"])
|
||||
print(f"\n{'='*50}")
|
||||
print(f"Summary: {passed}/{len(results)} tests passed")
|
||||
print(f"Pass Rate: {passed/len(results):.1%}")
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Running mock batch evaluation...")
|
||||
|
||||
# Mock evaluation without API key
|
||||
results = []
|
||||
for i, test_case in enumerate(test_cases):
|
||||
print(f"\nTest {i + 1}: {test_case['input']}")
|
||||
|
||||
# Mock result (all would fail without API key)
|
||||
mock_score = 0.0
|
||||
mock_passed = False
|
||||
|
||||
results.append({
|
||||
"input": test_case["input"],
|
||||
"expected": test_case["expected"],
|
||||
"score": mock_score,
|
||||
"passed": mock_passed
|
||||
})
|
||||
|
||||
print(f" Score: {mock_score}/10 (mock)")
|
||||
print(f" Passed: {mock_passed} (expected without API key)")
|
||||
|
||||
# Summary
|
||||
passed = sum(1 for r in results if r["passed"])
|
||||
print(f"\n{'='*50}")
|
||||
print(f"Summary: {passed}/{len(results)} tests passed (mock evaluation)")
|
||||
print(f"Pass Rate: {passed/len(results):.1%}")
|
||||
print("\nTo run real batch evaluation, set OPENAI_API_KEY environment variable")
|
||||
@@ -0,0 +1,82 @@
|
||||
"""
|
||||
Criteria Evaluation Example
|
||||
|
||||
This example demonstrates how to evaluate agent outputs
|
||||
against custom criteria using LLM-as-judge.
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.eval import CriteriaEvaluator
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
if has_api_key:
|
||||
print("--- Testing Criteria Evaluation with Agent ---")
|
||||
# Create a customer service agent
|
||||
agent = Agent(
|
||||
instructions="You are a friendly customer service agent. Be helpful and empathetic."
|
||||
)
|
||||
|
||||
# Create criteria evaluator with numeric scoring
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Response is helpful, empathetic, and provides a clear solution",
|
||||
agent=agent,
|
||||
input_text="I'm frustrated because my order hasn't arrived yet.",
|
||||
scoring_type="numeric", # Score 1-10
|
||||
threshold=7.0, # Pass if score >= 7
|
||||
num_iterations=2,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Run evaluation
|
||||
result = evaluator.run(print_summary=True)
|
||||
|
||||
print("\nNumeric Scoring Results:")
|
||||
print(f" Average Score: {result.avg_score}/10")
|
||||
print(f" Pass Rate: {result.pass_rate:.1%}")
|
||||
print(f" Passed: {result.passed}")
|
||||
|
||||
# Binary scoring example (pass/fail)
|
||||
print("\n--- Testing Binary Scoring ---")
|
||||
binary_evaluator = CriteriaEvaluator(
|
||||
criteria="Response does not contain any offensive language",
|
||||
agent=agent,
|
||||
input_text="Tell me a joke",
|
||||
scoring_type="binary", # Pass or Fail
|
||||
verbose=True
|
||||
)
|
||||
|
||||
binary_result = binary_evaluator.run(print_summary=True)
|
||||
|
||||
# With on_fail callback
|
||||
print("\n--- Testing Failure Callback ---")
|
||||
def handle_failure(score):
|
||||
print(f"ALERT: Evaluation failed with score {score.score}")
|
||||
print(f"Reasoning: {score.reasoning}")
|
||||
|
||||
callback_evaluator = CriteriaEvaluator(
|
||||
criteria="Response is professional and helpful",
|
||||
agent=agent,
|
||||
input_text="Help me",
|
||||
on_fail=handle_failure,
|
||||
threshold=8.0
|
||||
)
|
||||
|
||||
callback_evaluator.run()
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Skipping agent-based criteria evaluation...")
|
||||
|
||||
# Test pre-generated output evaluation (doesn't need agent)
|
||||
print("\n--- Testing Pre-generated Output Evaluation ---")
|
||||
if has_api_key:
|
||||
output = "I understand your frustration. Let me check on your order right away."
|
||||
try:
|
||||
result2 = evaluator.evaluate_output(output, print_summary=True)
|
||||
print(f"Pre-generated output score: {result2.avg_score}/10")
|
||||
except Exception as e:
|
||||
print(f"Pre-generated evaluation failed: {e}")
|
||||
else:
|
||||
print("⚠️ Skipping pre-generated evaluation (no API key)")
|
||||
print("To run full evaluation, set OPENAI_API_KEY environment variable")
|
||||
@@ -0,0 +1,59 @@
|
||||
"""
|
||||
Performance Evaluation Example
|
||||
|
||||
This example demonstrates how to benchmark agent performance
|
||||
by measuring runtime and memory usage across multiple iterations.
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.eval import PerformanceEvaluator
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
if has_api_key:
|
||||
print("--- Testing Agent Performance ---")
|
||||
# Create a simple agent
|
||||
agent = Agent(
|
||||
instructions="You are a helpful assistant. Answer questions briefly."
|
||||
)
|
||||
|
||||
# Create performance evaluator
|
||||
evaluator = PerformanceEvaluator(
|
||||
agent=agent,
|
||||
input_text="What is the capital of France?",
|
||||
num_iterations=10, # Run 10 benchmark iterations
|
||||
warmup_runs=2, # 2 warmup runs before measurement
|
||||
track_memory=True, # Track memory usage
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Run evaluation
|
||||
result = evaluator.run(print_summary=True)
|
||||
|
||||
# Access detailed metrics
|
||||
print("\nAgent Performance Results:")
|
||||
print(f" Average Time: {result.avg_run_time:.4f}s")
|
||||
print(f" Min Time: {result.min_run_time:.4f}s")
|
||||
print(f" Max Time: {result.max_run_time:.4f}s")
|
||||
print(f" Median Time: {result.median_run_time:.4f}s")
|
||||
print(f" P95 Time: {result.p95_run_time:.4f}s")
|
||||
print(f" Avg Memory: {result.avg_memory:.2f} MB")
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Skipping agent performance test...")
|
||||
|
||||
# You can also benchmark any function
|
||||
def my_function():
|
||||
import time
|
||||
time.sleep(0.1)
|
||||
return "done"
|
||||
|
||||
func_evaluator = PerformanceEvaluator(
|
||||
func=my_function,
|
||||
num_iterations=5,
|
||||
warmup_runs=1,
|
||||
track_memory=False
|
||||
)
|
||||
|
||||
func_result = func_evaluator.run(print_summary=True)
|
||||
@@ -0,0 +1,72 @@
|
||||
"""
|
||||
Reliability Evaluation Example
|
||||
|
||||
This example demonstrates how to verify that agents call
|
||||
the expected tools during execution.
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.eval import ReliabilityEvaluator
|
||||
|
||||
|
||||
def search_web(query: str) -> str:
|
||||
"""Search the web for information."""
|
||||
return f"Search results for: {query}"
|
||||
|
||||
|
||||
def calculate(expression: str) -> str:
|
||||
"""Calculate a math expression."""
|
||||
return str(eval(expression))
|
||||
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
if has_api_key:
|
||||
print("--- Testing Agent Tool Call Reliability ---")
|
||||
# Create an agent with tools
|
||||
agent = Agent(
|
||||
instructions="You are a helpful assistant with access to search and calculator tools.",
|
||||
tools=[search_web, calculate]
|
||||
)
|
||||
|
||||
# Create reliability evaluator
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="Search for the weather in Paris and calculate 25 * 4",
|
||||
expected_tools=["search_web", "calculate"], # Tools that should be called
|
||||
forbidden_tools=["delete_file"], # Tools that should NOT be called
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Run evaluation
|
||||
result = evaluator.run(print_summary=True)
|
||||
|
||||
# Check results
|
||||
print("\nAgent Reliability Results:")
|
||||
print(f" Passed: {result.passed}")
|
||||
print(f" Pass Rate: {result.pass_rate:.1%}")
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Skipping agent reliability test...")
|
||||
print("Agent would fail to call tools without API key (expected behavior)")
|
||||
|
||||
# You can also evaluate pre-recorded tool calls (doesn't need API key)
|
||||
print("\n--- Testing Pre-recorded Tool Call Evaluation ---")
|
||||
# Create a mock agent for tool call evaluation
|
||||
mock_agent = Agent(instructions="Mock agent")
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=mock_agent,
|
||||
expected_tools=["search_web", "calculate"], # Tools that should be called
|
||||
forbidden_tools=["delete_file"], # Tools that should NOT be called
|
||||
verbose=True
|
||||
)
|
||||
|
||||
result2 = evaluator.evaluate_tool_calls(
|
||||
actual_tools=["search_web", "calculate"],
|
||||
print_summary=True
|
||||
)
|
||||
|
||||
print("\nPre-recorded Tool Call Results:")
|
||||
print(f" Passed: {result2.passed}")
|
||||
print(f" Pass Rate: {result2.pass_rate:.1%}")
|
||||
@@ -0,0 +1,73 @@
|
||||
"""
|
||||
Basic Agent Skills Usage Example
|
||||
|
||||
This example demonstrates how to use Agent Skills with PraisonAI Agents.
|
||||
Skills are modular capabilities that can be loaded and used by agents.
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent, SkillManager
|
||||
|
||||
# Example 1: Using SkillManager directly
|
||||
print("=" * 50)
|
||||
print("Example 1: Using SkillManager directly")
|
||||
print("=" * 50)
|
||||
|
||||
# Create a skill manager
|
||||
manager = SkillManager()
|
||||
|
||||
# Discover skills from a directory
|
||||
# This scans the directory for subdirectories containing SKILL.md files
|
||||
skill_count = manager.discover(["./pdf-processing"], include_defaults=False)
|
||||
print(f"Discovered {skill_count} skill(s)")
|
||||
|
||||
# List available skills
|
||||
for skill in manager:
|
||||
print(f" - {skill.properties.name}: {skill.properties.description[:50]}...")
|
||||
|
||||
# Generate prompt XML for system prompt injection
|
||||
prompt_xml = manager.to_prompt()
|
||||
print("\nGenerated prompt XML:")
|
||||
print(prompt_xml)
|
||||
|
||||
# Example 2: Using skills with an Agent
|
||||
print("\n" + "=" * 50)
|
||||
print("Example 2: Using skills with an Agent")
|
||||
print("=" * 50)
|
||||
|
||||
# Create an agent with skills
|
||||
# Skills are lazy-loaded only when accessed
|
||||
agent = Agent(
|
||||
name="PDF Assistant",
|
||||
instructions="You are a helpful assistant that can process PDF documents.",
|
||||
skills=["./pdf-processing"], # Direct skill paths
|
||||
)
|
||||
|
||||
# Access the skill manager through the agent
|
||||
if agent.skill_manager:
|
||||
print(f"Agent has {len(agent.skill_manager)} skill(s) loaded")
|
||||
|
||||
# Get the skills prompt for system prompt injection
|
||||
skills_prompt = agent.get_skills_prompt()
|
||||
print("\nAgent skills prompt:")
|
||||
print(skills_prompt)
|
||||
|
||||
# Example 3: Discovering skills from multiple directories
|
||||
print("\n" + "=" * 50)
|
||||
print("Example 3: Using skills_dirs for discovery")
|
||||
print("=" * 50)
|
||||
|
||||
# Create an agent that discovers skills from directories
|
||||
agent_with_discovery = Agent(
|
||||
name="Multi-Skill Agent",
|
||||
instructions="You are a versatile assistant with multiple skills.",
|
||||
skills_dirs=["./"], # Scan current directory for skill subdirectories
|
||||
)
|
||||
|
||||
if agent_with_discovery.skill_manager:
|
||||
print(f"Discovered {len(agent_with_discovery.skill_manager)} skill(s)")
|
||||
for name in agent_with_discovery.skill_manager.skill_names:
|
||||
print(f" - {name}")
|
||||
|
||||
print("\n" + "=" * 50)
|
||||
print("Examples complete!")
|
||||
print("=" * 50)
|
||||
@@ -0,0 +1,163 @@
|
||||
"""
|
||||
Custom Skill Creation Example
|
||||
|
||||
This example demonstrates how to programmatically create and validate
|
||||
custom Agent Skills using PraisonAI Agents.
|
||||
"""
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from praisonaiagents.skills import (
|
||||
SkillManager,
|
||||
SkillLoader,
|
||||
validate,
|
||||
read_properties,
|
||||
)
|
||||
|
||||
|
||||
def create_custom_skill(
|
||||
name: str,
|
||||
description: str,
|
||||
instructions: str,
|
||||
base_dir: str = None
|
||||
) -> Path:
|
||||
"""Create a custom skill programmatically.
|
||||
|
||||
Args:
|
||||
name: Skill name (kebab-case)
|
||||
description: What the skill does
|
||||
instructions: Markdown instructions for the skill
|
||||
base_dir: Directory to create skill in (default: temp dir)
|
||||
|
||||
Returns:
|
||||
Path to the created skill directory
|
||||
"""
|
||||
if base_dir is None:
|
||||
base_dir = tempfile.mkdtemp()
|
||||
|
||||
skill_dir = Path(base_dir) / name
|
||||
skill_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create SKILL.md
|
||||
skill_md_content = f"""---
|
||||
name: {name}
|
||||
description: {description}
|
||||
license: Apache-2.0
|
||||
metadata:
|
||||
author: custom
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
{instructions}
|
||||
"""
|
||||
(skill_dir / "SKILL.md").write_text(skill_md_content)
|
||||
|
||||
# Create optional directories
|
||||
(skill_dir / "scripts").mkdir(exist_ok=True)
|
||||
(skill_dir / "references").mkdir(exist_ok=True)
|
||||
(skill_dir / "assets").mkdir(exist_ok=True)
|
||||
|
||||
return skill_dir
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 60)
|
||||
print("Custom Skill Creation Example")
|
||||
print("=" * 60)
|
||||
|
||||
# Create a custom skill
|
||||
print("\n1. Creating a custom skill...")
|
||||
|
||||
skill_path = create_custom_skill(
|
||||
name="code-review",
|
||||
description="Review code for best practices, bugs, and improvements. Use when asked to review or analyze code quality.",
|
||||
instructions="""
|
||||
# Code Review Skill
|
||||
|
||||
## Overview
|
||||
This skill enables thorough code review with focus on:
|
||||
- Code quality and readability
|
||||
- Potential bugs and edge cases
|
||||
- Performance considerations
|
||||
- Security vulnerabilities
|
||||
- Best practices adherence
|
||||
|
||||
## Instructions
|
||||
|
||||
When reviewing code:
|
||||
|
||||
1. **Read the code carefully** - Understand the purpose and flow
|
||||
2. **Check for bugs** - Look for null checks, edge cases, error handling
|
||||
3. **Evaluate readability** - Variable names, comments, structure
|
||||
4. **Consider performance** - Time/space complexity, unnecessary operations
|
||||
5. **Security review** - Input validation, injection risks, data exposure
|
||||
6. **Suggest improvements** - Provide actionable recommendations
|
||||
|
||||
## Output Format
|
||||
|
||||
Structure your review as:
|
||||
- Summary: Brief overview of the code
|
||||
- Issues: List of problems found (severity: high/medium/low)
|
||||
- Suggestions: Recommended improvements
|
||||
- Positive aspects: What's done well
|
||||
"""
|
||||
)
|
||||
|
||||
print(f" Created skill at: {skill_path}")
|
||||
|
||||
# Validate the skill
|
||||
print("\n2. Validating the skill...")
|
||||
errors = validate(skill_path)
|
||||
|
||||
if errors:
|
||||
print(" Validation errors:")
|
||||
for error in errors:
|
||||
print(f" - {error}")
|
||||
else:
|
||||
print(" ✓ Skill is valid!")
|
||||
|
||||
# Load and inspect the skill
|
||||
print("\n3. Loading skill properties...")
|
||||
props = read_properties(skill_path)
|
||||
print(f" Name: {props.name}")
|
||||
print(f" Description: {props.description[:60]}...")
|
||||
print(f" License: {props.license}")
|
||||
|
||||
# Use with SkillLoader for progressive loading
|
||||
print("\n4. Using SkillLoader for progressive disclosure...")
|
||||
loader = SkillLoader()
|
||||
|
||||
# Level 1: Load metadata only
|
||||
skill = loader.load_metadata(str(skill_path))
|
||||
print(f" Level 1 - Metadata loaded: {skill.properties.name}")
|
||||
print(f" Instructions loaded: {skill.is_activated}")
|
||||
|
||||
# Level 2: Activate (load instructions)
|
||||
loader.activate(skill)
|
||||
print(f" Level 2 - Activated: {skill.is_activated}")
|
||||
print(f" Instructions preview: {skill.instructions[:100]}...")
|
||||
|
||||
# Use with SkillManager
|
||||
print("\n5. Using with SkillManager...")
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_path))
|
||||
|
||||
# Generate prompt XML
|
||||
prompt = manager.to_prompt()
|
||||
print(" Generated prompt XML:")
|
||||
print(prompt)
|
||||
|
||||
# Clean up
|
||||
print("\n6. Cleanup...")
|
||||
import shutil
|
||||
shutil.rmtree(skill_path.parent)
|
||||
print(" Temporary files removed.")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Example complete!")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,38 @@
|
||||
---
|
||||
name: pdf-processing
|
||||
description: Process and extract information from PDF documents. Use this skill when the user asks to read, analyze, or extract data from PDF files.
|
||||
license: Apache-2.0
|
||||
compatibility: Works with PraisonAI Agents
|
||||
metadata:
|
||||
author: praisonai
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
# PDF Processing Skill
|
||||
|
||||
## Overview
|
||||
|
||||
This skill enables agents to process PDF documents, extract text content, and analyze the information within them.
|
||||
|
||||
## When to Use
|
||||
|
||||
Activate this skill when:
|
||||
- User asks to read or analyze a PDF file
|
||||
- User needs to extract text from a PDF
|
||||
- User wants to summarize a PDF document
|
||||
- User needs to search for information within a PDF
|
||||
|
||||
## Instructions
|
||||
|
||||
1. First, verify the PDF file exists at the specified path
|
||||
2. Use appropriate tools to read the PDF content
|
||||
3. Extract text while preserving structure where possible
|
||||
4. For large PDFs, process in chunks to manage context
|
||||
5. Summarize or analyze based on user's specific request
|
||||
|
||||
## Best Practices
|
||||
|
||||
- Always confirm the file path before processing
|
||||
- Handle encrypted PDFs gracefully with appropriate error messages
|
||||
- For scanned PDFs, note that OCR may be required
|
||||
- Preserve important formatting like tables and lists
|
||||
@@ -3,7 +3,7 @@ Praison AI Agents - A package for hierarchical AI agent task execution
|
||||
"""
|
||||
|
||||
# Apply warning patch BEFORE any imports to intercept warnings at the source
|
||||
from . import _warning_patch
|
||||
from . import _warning_patch # noqa: F401
|
||||
|
||||
# Import centralized logging configuration FIRST
|
||||
from . import _logging
|
||||
@@ -78,6 +78,19 @@ def __getattr__(name):
|
||||
elif name == "LineRange":
|
||||
from praisonaiagents.context.fast import LineRange
|
||||
return LineRange
|
||||
# Agent Skills support (lazy loaded for zero performance impact)
|
||||
elif name == "SkillManager":
|
||||
from praisonaiagents.skills import SkillManager
|
||||
return SkillManager
|
||||
elif name == "SkillProperties":
|
||||
from praisonaiagents.skills import SkillProperties
|
||||
return SkillProperties
|
||||
elif name == "SkillMetadata":
|
||||
from praisonaiagents.skills import SkillMetadata
|
||||
return SkillMetadata
|
||||
elif name == "SkillLoader":
|
||||
from praisonaiagents.skills import SkillLoader
|
||||
return SkillLoader
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
# Planning mode support
|
||||
@@ -289,6 +302,23 @@ __all__ = [
|
||||
'ExpandResult'
|
||||
]
|
||||
|
||||
# Add workflow exports
|
||||
__all__.extend([
|
||||
'Workflow',
|
||||
'WorkflowStep',
|
||||
'WorkflowContext',
|
||||
'StepResult',
|
||||
'Route',
|
||||
'Parallel',
|
||||
'Loop',
|
||||
'Repeat',
|
||||
'route',
|
||||
'parallel',
|
||||
'loop',
|
||||
'repeat',
|
||||
'Pipeline'
|
||||
])
|
||||
|
||||
# Add MCP to __all__ if available
|
||||
if _mcp_available:
|
||||
__all__.append('MCP')
|
||||
@@ -305,6 +335,14 @@ __all__.extend([
|
||||
'LineRange'
|
||||
])
|
||||
|
||||
# Add Agent Skills exports (lazy loaded)
|
||||
__all__.extend([
|
||||
'SkillManager',
|
||||
'SkillProperties',
|
||||
'SkillMetadata',
|
||||
'SkillLoader'
|
||||
])
|
||||
|
||||
# AG-UI support (optional, lazy loaded)
|
||||
try:
|
||||
from praisonaiagents.ui.agui import AGUI
|
||||
|
||||
@@ -8,6 +8,7 @@ import functools
|
||||
import sys
|
||||
|
||||
# Apply aggressive warning filters first
|
||||
warnings.filterwarnings("ignore", message=".*There is no current event loop.*", category=DeprecationWarning)
|
||||
warnings.filterwarnings("ignore", message=".*Pydantic serializer warnings.*")
|
||||
warnings.filterwarnings("ignore", message=".*PydanticSerializationUnexpectedValue.*")
|
||||
warnings.filterwarnings("ignore", message=".*Expected 9 fields but got.*")
|
||||
@@ -31,7 +32,8 @@ SUPPRESSED_PATTERNS = [
|
||||
"Expected `StreamingChoices` but got `Choices`",
|
||||
"serialized value may not be as expected",
|
||||
"Mixing V1 models and V2 models",
|
||||
"Please upgrade `Settings` to V2"
|
||||
"Please upgrade `Settings` to V2",
|
||||
"There is no current event loop"
|
||||
]
|
||||
|
||||
@functools.wraps(_original_warn)
|
||||
@@ -43,7 +45,7 @@ def _patched_warn(message, category=None, stacklevel=1, source=None):
|
||||
if pattern in msg_str:
|
||||
return
|
||||
|
||||
if category == UserWarning and "pydantic" in msg_str.lower():
|
||||
if category is UserWarning and "pydantic" in msg_str.lower():
|
||||
return
|
||||
|
||||
_original_warn(message, category, stacklevel, source)
|
||||
@@ -57,7 +59,7 @@ def _patched_warn_explicit(message, category, filename, lineno, module=None, reg
|
||||
if pattern in msg_str:
|
||||
return
|
||||
|
||||
if category == UserWarning and "pydantic" in msg_str.lower():
|
||||
if category is UserWarning and "pydantic" in msg_str.lower():
|
||||
return
|
||||
|
||||
if module and "pydantic" in str(module):
|
||||
|
||||
@@ -250,7 +250,9 @@ class Agent:
|
||||
fast_context_parallelism: int = 8,
|
||||
fast_context_timeout: float = 30.0,
|
||||
history_in_context: Optional[int] = None,
|
||||
auto_save: Optional[str] = None
|
||||
auto_save: Optional[str] = None,
|
||||
skills: Optional[List[str]] = None,
|
||||
skills_dirs: Optional[List[str]] = None
|
||||
):
|
||||
"""Initialize an Agent instance.
|
||||
|
||||
@@ -607,6 +609,12 @@ Your Goal: {self.goal}
|
||||
self.fast_context_timeout = fast_context_timeout
|
||||
self._fast_context_instance = None # Lazy loaded
|
||||
|
||||
# Agent Skills configuration (lazy loaded for zero performance impact)
|
||||
self._skills = skills
|
||||
self._skills_dirs = skills_dirs
|
||||
self._skill_manager = None # Lazy loaded
|
||||
self._skills_initialized = False
|
||||
|
||||
@property
|
||||
def console(self):
|
||||
"""Lazily initialize Rich Console only when needed."""
|
||||
@@ -626,6 +634,35 @@ Your Goal: {self.goal}
|
||||
def fast_context_path(self, value):
|
||||
self._fast_context_path = value
|
||||
|
||||
@property
|
||||
def skill_manager(self):
|
||||
"""Lazily initialize SkillManager only when skills are accessed."""
|
||||
if self._skill_manager is None and (self._skills or self._skills_dirs):
|
||||
from ..skills import SkillManager
|
||||
self._skill_manager = SkillManager()
|
||||
|
||||
# Add explicit skill paths
|
||||
if self._skills:
|
||||
for skill_path in self._skills:
|
||||
self._skill_manager.add_skill(skill_path)
|
||||
|
||||
# Discover skills from directories
|
||||
if self._skills_dirs:
|
||||
self._skill_manager.discover(self._skills_dirs, include_defaults=False)
|
||||
|
||||
self._skills_initialized = True
|
||||
return self._skill_manager
|
||||
|
||||
def get_skills_prompt(self) -> str:
|
||||
"""Get the XML prompt for available skills.
|
||||
|
||||
Returns:
|
||||
XML string with <available_skills> block, or empty string if no skills
|
||||
"""
|
||||
if self.skill_manager is None:
|
||||
return ""
|
||||
return self.skill_manager.to_prompt()
|
||||
|
||||
@property
|
||||
def _openai_client(self):
|
||||
"""Lazily initialize OpenAI client only when needed."""
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
"""
|
||||
PraisonAI Agents Evaluation Framework.
|
||||
|
||||
Provides comprehensive evaluation capabilities for AI agents with zero performance
|
||||
impact when not in use through lazy loading.
|
||||
|
||||
Evaluator Types:
|
||||
- AccuracyEvaluator: Compare output against expected output using LLM-as-judge
|
||||
- PerformanceEvaluator: Measure runtime and memory usage
|
||||
- ReliabilityEvaluator: Verify expected tool calls are made
|
||||
- CriteriaEvaluator: Evaluate against custom criteria
|
||||
|
||||
Example:
|
||||
>>> from praisonaiagents.eval import AccuracyEvaluator
|
||||
>>> evaluator = AccuracyEvaluator(
|
||||
... agent=my_agent,
|
||||
... input_text="What is 2+2?",
|
||||
... expected_output="4"
|
||||
... )
|
||||
>>> result = evaluator.run(print_summary=True)
|
||||
"""
|
||||
|
||||
__all__ = [
|
||||
"BaseEvaluator",
|
||||
"AccuracyEvaluator",
|
||||
"PerformanceEvaluator",
|
||||
"ReliabilityEvaluator",
|
||||
"CriteriaEvaluator",
|
||||
"AccuracyResult",
|
||||
"PerformanceResult",
|
||||
"ReliabilityResult",
|
||||
"CriteriaResult",
|
||||
"EvaluationScore",
|
||||
"PerformanceMetrics",
|
||||
"ToolCallResult",
|
||||
"CriteriaScore",
|
||||
]
|
||||
|
||||
_LAZY_IMPORTS = {
|
||||
"BaseEvaluator": ("base", "BaseEvaluator"),
|
||||
"AccuracyEvaluator": ("accuracy", "AccuracyEvaluator"),
|
||||
"PerformanceEvaluator": ("performance", "PerformanceEvaluator"),
|
||||
"ReliabilityEvaluator": ("reliability", "ReliabilityEvaluator"),
|
||||
"CriteriaEvaluator": ("criteria", "CriteriaEvaluator"),
|
||||
"AccuracyResult": ("results", "AccuracyResult"),
|
||||
"PerformanceResult": ("results", "PerformanceResult"),
|
||||
"ReliabilityResult": ("results", "ReliabilityResult"),
|
||||
"CriteriaResult": ("results", "CriteriaResult"),
|
||||
"EvaluationScore": ("results", "EvaluationScore"),
|
||||
"PerformanceMetrics": ("results", "PerformanceMetrics"),
|
||||
"ToolCallResult": ("results", "ToolCallResult"),
|
||||
"CriteriaScore": ("results", "CriteriaScore"),
|
||||
}
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
"""Lazy import mechanism for zero-cost imports when not used."""
|
||||
if name in _LAZY_IMPORTS:
|
||||
module_name, attr_name = _LAZY_IMPORTS[name]
|
||||
import importlib
|
||||
module = importlib.import_module(f".{module_name}", __name__)
|
||||
return getattr(module, attr_name)
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
def __dir__():
|
||||
"""Return list of available attributes for tab completion."""
|
||||
return __all__
|
||||
@@ -0,0 +1,239 @@
|
||||
"""
|
||||
Accuracy evaluator for PraisonAI Agents.
|
||||
|
||||
Evaluates agent output accuracy by comparing against expected output using LLM-as-judge.
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from typing import Any, Callable, List, Optional, Union, TYPE_CHECKING
|
||||
|
||||
from .base import BaseEvaluator
|
||||
from .results import AccuracyResult, EvaluationScore
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..agent.agent import Agent
|
||||
from ..agents.agents import PraisonAIAgents
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AccuracyEvaluator(BaseEvaluator):
|
||||
"""
|
||||
Evaluates the accuracy of agent outputs against expected outputs.
|
||||
|
||||
Uses an LLM as a judge to score outputs on a scale of 1-10.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
agent: Optional[Union["Agent", "PraisonAIAgents"]] = None,
|
||||
func: Optional[Callable[..., str]] = None,
|
||||
input_text: str = "",
|
||||
expected_output: str = "",
|
||||
num_iterations: int = 1,
|
||||
model: Optional[str] = None,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the accuracy evaluator.
|
||||
|
||||
Args:
|
||||
agent: Agent or PraisonAIAgents instance to evaluate
|
||||
func: Alternative callable that returns output string
|
||||
input_text: Input to provide to the agent/function
|
||||
expected_output: Expected output to compare against
|
||||
num_iterations: Number of evaluation iterations
|
||||
model: LLM model to use for judging (defaults to gpt-4o-mini)
|
||||
name: Name for this evaluation
|
||||
save_results_path: Path to save results
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
super().__init__(name=name, save_results_path=save_results_path, verbose=verbose)
|
||||
|
||||
self.agent = agent
|
||||
self.func = func
|
||||
self.input_text = input_text
|
||||
self.expected_output = expected_output
|
||||
self.num_iterations = num_iterations
|
||||
self.model = model or os.getenv("OPENAI_MODEL_NAME", "gpt-4o-mini")
|
||||
|
||||
if agent is None and func is None:
|
||||
raise ValueError("Either 'agent' or 'func' must be provided")
|
||||
|
||||
def _get_output(self) -> str:
|
||||
"""Get output from agent or function."""
|
||||
if self.func:
|
||||
return str(self.func(self.input_text))
|
||||
|
||||
if hasattr(self.agent, 'chat'):
|
||||
return str(self.agent.chat(self.input_text))
|
||||
elif hasattr(self.agent, 'start'):
|
||||
result = self.agent.start(self.input_text)
|
||||
if hasattr(result, 'raw'):
|
||||
return str(result.raw)
|
||||
return str(result)
|
||||
else:
|
||||
raise ValueError("Agent must have 'chat' or 'start' method")
|
||||
|
||||
def _judge_output(self, output: str) -> EvaluationScore:
|
||||
"""
|
||||
Use LLM to judge the output against expected output.
|
||||
|
||||
Args:
|
||||
output: The actual output to evaluate
|
||||
|
||||
Returns:
|
||||
EvaluationScore with score and reasoning
|
||||
"""
|
||||
try:
|
||||
from openai import OpenAI
|
||||
except ImportError:
|
||||
raise ImportError("OpenAI package required for accuracy evaluation. Install with: pip install openai")
|
||||
|
||||
client = OpenAI()
|
||||
|
||||
prompt = f"""You are an expert evaluator. Compare the actual output against the expected output and provide a score from 1-10.
|
||||
|
||||
Input: {self.input_text}
|
||||
|
||||
Expected Output: {self.expected_output}
|
||||
|
||||
Actual Output: {output}
|
||||
|
||||
Scoring Guidelines:
|
||||
- 10: Perfect match in meaning and completeness
|
||||
- 8-9: Very close, minor differences that don't affect correctness
|
||||
- 6-7: Mostly correct but missing some details or has minor errors
|
||||
- 4-5: Partially correct but significant issues
|
||||
- 2-3: Mostly incorrect but shows some understanding
|
||||
- 1: Completely wrong or irrelevant
|
||||
|
||||
Respond in this exact format:
|
||||
SCORE: [number 1-10]
|
||||
REASONING: [brief explanation]"""
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.1,
|
||||
max_tokens=200
|
||||
)
|
||||
|
||||
response_text = response.choices[0].message.content or ""
|
||||
|
||||
score = 5.0
|
||||
reasoning = "Unable to parse response"
|
||||
|
||||
lines = response_text.strip().split('\n')
|
||||
for line in lines:
|
||||
if line.startswith('SCORE:'):
|
||||
try:
|
||||
score = float(line.replace('SCORE:', '').strip())
|
||||
score = max(1.0, min(10.0, score))
|
||||
except ValueError:
|
||||
pass
|
||||
elif line.startswith('REASONING:'):
|
||||
reasoning = line.replace('REASONING:', '').strip()
|
||||
|
||||
return EvaluationScore(
|
||||
score=score,
|
||||
reasoning=reasoning,
|
||||
input_text=self.input_text,
|
||||
output_text=output,
|
||||
expected_output=self.expected_output
|
||||
)
|
||||
|
||||
def run(self, print_summary: bool = False) -> AccuracyResult:
|
||||
"""
|
||||
Execute the accuracy evaluation.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
AccuracyResult with all evaluation scores
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = AccuracyResult(
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
for i in range(self.num_iterations):
|
||||
if self.verbose:
|
||||
logger.info(f"Running iteration {i + 1}/{self.num_iterations}")
|
||||
|
||||
try:
|
||||
output = self._get_output()
|
||||
score = self._judge_output(output)
|
||||
result.evaluations.append(score)
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f" Score: {score.score}/10 - {score.reasoning}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error in iteration {i + 1}: {e}")
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=0.0,
|
||||
reasoning=f"Error: {str(e)}",
|
||||
input_text=self.input_text,
|
||||
output_text="",
|
||||
expected_output=self.expected_output
|
||||
))
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
async def run_async(self, print_summary: bool = False) -> AccuracyResult:
|
||||
"""
|
||||
Execute the accuracy evaluation asynchronously.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
AccuracyResult with all evaluation scores
|
||||
"""
|
||||
await self.async_before_run()
|
||||
result = self.run(print_summary=False)
|
||||
await self.async_after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
def evaluate_output(self, output: str, print_summary: bool = False) -> AccuracyResult:
|
||||
"""
|
||||
Evaluate a pre-generated output without running the agent.
|
||||
|
||||
Args:
|
||||
output: The output to evaluate
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
AccuracyResult with evaluation score
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = AccuracyResult(
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
score = self._judge_output(output)
|
||||
result.evaluations.append(score)
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,143 @@
|
||||
"""
|
||||
Base evaluator class for PraisonAI Agents evaluation framework.
|
||||
|
||||
This module provides the abstract base class that all evaluators inherit from.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Dict, Optional, Union, TYPE_CHECKING
|
||||
import uuid
|
||||
import logging
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .results import AccuracyResult, PerformanceResult, ReliabilityResult, CriteriaResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BaseEvaluator(ABC):
|
||||
"""
|
||||
Abstract base class for all evaluators in PraisonAI Agents.
|
||||
|
||||
Provides common functionality for evaluation lifecycle management,
|
||||
result storage, and telemetry integration.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the base evaluator.
|
||||
|
||||
Args:
|
||||
name: Optional name for this evaluation run
|
||||
save_results_path: Optional file path to save results (supports {name}, {eval_id} placeholders)
|
||||
verbose: Enable verbose output during evaluation
|
||||
"""
|
||||
self.eval_id = str(uuid.uuid4())[:8]
|
||||
self.name = name or f"eval_{self.eval_id}"
|
||||
self.save_results_path = save_results_path
|
||||
self.verbose = verbose
|
||||
self._result = None
|
||||
|
||||
def before_run(self) -> None:
|
||||
"""
|
||||
Hook called before evaluation runs.
|
||||
Override in subclasses for custom pre-evaluation logic.
|
||||
"""
|
||||
if self.verbose:
|
||||
logger.info(f"Starting evaluation: {self.name} (ID: {self.eval_id})")
|
||||
|
||||
def after_run(self, result: Any) -> None:
|
||||
"""
|
||||
Hook called after evaluation completes.
|
||||
Override in subclasses for custom post-evaluation logic.
|
||||
|
||||
Args:
|
||||
result: The evaluation result
|
||||
"""
|
||||
if self.verbose:
|
||||
logger.info(f"Completed evaluation: {self.name}")
|
||||
|
||||
if self.save_results_path:
|
||||
self._save_result(result)
|
||||
|
||||
async def async_before_run(self) -> None:
|
||||
"""
|
||||
Async hook called before evaluation runs.
|
||||
Override in subclasses for custom async pre-evaluation logic.
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
async def async_after_run(self, result: Any) -> None:
|
||||
"""
|
||||
Async hook called after evaluation completes.
|
||||
Override in subclasses for custom async post-evaluation logic.
|
||||
|
||||
Args:
|
||||
result: The evaluation result
|
||||
"""
|
||||
self.after_run(result)
|
||||
|
||||
@abstractmethod
|
||||
def run(self, **kwargs) -> Any:
|
||||
"""
|
||||
Execute the evaluation synchronously.
|
||||
|
||||
Returns:
|
||||
Evaluation result (type depends on evaluator)
|
||||
"""
|
||||
pass
|
||||
|
||||
async def run_async(self, **kwargs) -> Any:
|
||||
"""
|
||||
Execute the evaluation asynchronously.
|
||||
Default implementation calls sync run().
|
||||
Override for true async evaluation.
|
||||
|
||||
Returns:
|
||||
Evaluation result (type depends on evaluator)
|
||||
"""
|
||||
return self.run(**kwargs)
|
||||
|
||||
def _save_result(self, result: Any) -> None:
|
||||
"""
|
||||
Save evaluation result to file.
|
||||
|
||||
Args:
|
||||
result: The evaluation result to save
|
||||
"""
|
||||
if not self.save_results_path:
|
||||
return
|
||||
|
||||
try:
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
file_path = self.save_results_path.format(
|
||||
name=self.name,
|
||||
eval_id=self.eval_id
|
||||
)
|
||||
|
||||
path = Path(file_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if hasattr(result, 'to_dict'):
|
||||
data = result.to_dict()
|
||||
elif hasattr(result, '__dict__'):
|
||||
data = result.__dict__
|
||||
else:
|
||||
data = {"result": str(result)}
|
||||
|
||||
path.write_text(json.dumps(data, indent=2, default=str))
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f"Saved results to: {file_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save results: {e}")
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"{self.__class__.__name__}(name={self.name!r}, eval_id={self.eval_id!r})"
|
||||
@@ -0,0 +1,289 @@
|
||||
"""
|
||||
Criteria evaluator for PraisonAI Agents.
|
||||
|
||||
Evaluates agent outputs against custom criteria using LLM-as-judge.
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from typing import Callable, List, Literal, Optional, Union, TYPE_CHECKING
|
||||
|
||||
from .base import BaseEvaluator
|
||||
from .results import CriteriaResult, CriteriaScore
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..agent.agent import Agent
|
||||
from ..agents.agents import PraisonAIAgents
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CriteriaEvaluator(BaseEvaluator):
|
||||
"""
|
||||
Evaluates agent outputs against custom criteria.
|
||||
|
||||
Uses an LLM as a judge to score outputs based on user-defined criteria.
|
||||
Supports both numeric (1-10) and binary (pass/fail) scoring.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
criteria: str,
|
||||
agent: Optional[Union["Agent", "PraisonAIAgents"]] = None,
|
||||
func: Optional[Callable[..., str]] = None,
|
||||
input_text: str = "",
|
||||
scoring_type: Literal["numeric", "binary"] = "numeric",
|
||||
threshold: float = 7.0,
|
||||
num_iterations: int = 1,
|
||||
model: Optional[str] = None,
|
||||
on_fail: Optional[Callable[[CriteriaScore], None]] = None,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the criteria evaluator.
|
||||
|
||||
Args:
|
||||
criteria: The criteria to evaluate against (e.g., "Response is helpful and accurate")
|
||||
agent: Agent or PraisonAIAgents instance to evaluate
|
||||
func: Alternative callable that returns output string
|
||||
input_text: Input to provide to the agent/function
|
||||
scoring_type: "numeric" (1-10 scale) or "binary" (pass/fail)
|
||||
threshold: Score threshold for passing (numeric mode only)
|
||||
num_iterations: Number of evaluation iterations
|
||||
model: LLM model to use for judging (defaults to gpt-4o-mini)
|
||||
on_fail: Callback function called when evaluation fails
|
||||
name: Name for this evaluation
|
||||
save_results_path: Path to save results
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
super().__init__(name=name, save_results_path=save_results_path, verbose=verbose)
|
||||
|
||||
self.criteria = criteria
|
||||
self.agent = agent
|
||||
self.func = func
|
||||
self.input_text = input_text
|
||||
self.scoring_type = scoring_type
|
||||
self.threshold = threshold
|
||||
self.num_iterations = num_iterations
|
||||
self.model = model or os.getenv("OPENAI_MODEL_NAME", "gpt-4o-mini")
|
||||
self.on_fail = on_fail
|
||||
|
||||
def _get_output(self) -> str:
|
||||
"""Get output from agent or function."""
|
||||
if self.func:
|
||||
return str(self.func(self.input_text))
|
||||
|
||||
if self.agent is None:
|
||||
raise ValueError("Either 'agent' or 'func' must be provided")
|
||||
|
||||
if hasattr(self.agent, 'chat'):
|
||||
return str(self.agent.chat(self.input_text))
|
||||
elif hasattr(self.agent, 'start'):
|
||||
result = self.agent.start(self.input_text)
|
||||
if hasattr(result, 'raw'):
|
||||
return str(result.raw)
|
||||
return str(result)
|
||||
else:
|
||||
raise ValueError("Agent must have 'chat' or 'start' method")
|
||||
|
||||
def _judge_output(self, output: str) -> CriteriaScore:
|
||||
"""
|
||||
Use LLM to judge the output against criteria.
|
||||
|
||||
Args:
|
||||
output: The output to evaluate
|
||||
|
||||
Returns:
|
||||
CriteriaScore with score, passed status, and reasoning
|
||||
"""
|
||||
try:
|
||||
from openai import OpenAI
|
||||
except ImportError:
|
||||
raise ImportError("OpenAI package required for criteria evaluation. Install with: pip install openai")
|
||||
|
||||
client = OpenAI()
|
||||
|
||||
if self.scoring_type == "binary":
|
||||
prompt = f"""You are an expert evaluator. Evaluate the following output against the given criteria.
|
||||
|
||||
Criteria: {self.criteria}
|
||||
|
||||
Output to evaluate:
|
||||
{output}
|
||||
|
||||
Determine if the output PASSES or FAILS the criteria.
|
||||
|
||||
Respond in this exact format:
|
||||
RESULT: [PASS or FAIL]
|
||||
REASONING: [brief explanation]"""
|
||||
else:
|
||||
prompt = f"""You are an expert evaluator. Evaluate the following output against the given criteria.
|
||||
|
||||
Criteria: {self.criteria}
|
||||
|
||||
Output to evaluate:
|
||||
{output}
|
||||
|
||||
Score the output from 1-10 based on how well it meets the criteria.
|
||||
- 10: Perfectly meets all criteria
|
||||
- 8-9: Meets criteria very well with minor issues
|
||||
- 6-7: Meets most criteria but has some gaps
|
||||
- 4-5: Partially meets criteria
|
||||
- 2-3: Barely meets criteria
|
||||
- 1: Does not meet criteria at all
|
||||
|
||||
Respond in this exact format:
|
||||
SCORE: [number 1-10]
|
||||
REASONING: [brief explanation]"""
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.1,
|
||||
max_tokens=200
|
||||
)
|
||||
|
||||
response_text = response.choices[0].message.content or ""
|
||||
|
||||
score = 5.0
|
||||
passed = False
|
||||
reasoning = "Unable to parse response"
|
||||
|
||||
lines = response_text.strip().split('\n')
|
||||
for line in lines:
|
||||
if self.scoring_type == "binary":
|
||||
if line.startswith('RESULT:'):
|
||||
result_str = line.replace('RESULT:', '').strip().upper()
|
||||
passed = result_str == 'PASS'
|
||||
score = 10.0 if passed else 0.0
|
||||
else:
|
||||
if line.startswith('SCORE:'):
|
||||
try:
|
||||
score = float(line.replace('SCORE:', '').strip())
|
||||
score = max(1.0, min(10.0, score))
|
||||
passed = score >= self.threshold
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if line.startswith('REASONING:'):
|
||||
reasoning = line.replace('REASONING:', '').strip()
|
||||
|
||||
return CriteriaScore(
|
||||
score=score,
|
||||
passed=passed,
|
||||
reasoning=reasoning,
|
||||
output_text=output,
|
||||
criteria=self.criteria
|
||||
)
|
||||
|
||||
def run(self, print_summary: bool = False) -> CriteriaResult:
|
||||
"""
|
||||
Execute the criteria evaluation.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
CriteriaResult with all evaluation scores
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = CriteriaResult(
|
||||
criteria=self.criteria,
|
||||
scoring_type=self.scoring_type,
|
||||
threshold=self.threshold,
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
for i in range(self.num_iterations):
|
||||
if self.verbose:
|
||||
logger.info(f"Running iteration {i + 1}/{self.num_iterations}")
|
||||
|
||||
try:
|
||||
output = self._get_output()
|
||||
score = self._judge_output(output)
|
||||
result.evaluations.append(score)
|
||||
|
||||
if self.verbose:
|
||||
status = "PASS" if score.passed else "FAIL"
|
||||
logger.info(f" {status} (Score: {score.score}) - {score.reasoning}")
|
||||
|
||||
if not score.passed and self.on_fail:
|
||||
self.on_fail(score)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in iteration {i + 1}: {e}")
|
||||
error_score = CriteriaScore(
|
||||
score=0.0,
|
||||
passed=False,
|
||||
reasoning=f"Error: {str(e)}",
|
||||
output_text="",
|
||||
criteria=self.criteria
|
||||
)
|
||||
result.evaluations.append(error_score)
|
||||
|
||||
if self.on_fail:
|
||||
self.on_fail(error_score)
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
async def run_async(self, print_summary: bool = False) -> CriteriaResult:
|
||||
"""
|
||||
Execute the criteria evaluation asynchronously.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
CriteriaResult with all evaluation scores
|
||||
"""
|
||||
await self.async_before_run()
|
||||
result = self.run(print_summary=False)
|
||||
await self.async_after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
def evaluate_output(self, output: str, print_summary: bool = False) -> CriteriaResult:
|
||||
"""
|
||||
Evaluate a pre-generated output without running the agent.
|
||||
|
||||
Args:
|
||||
output: The output to evaluate
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
CriteriaResult with evaluation score
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = CriteriaResult(
|
||||
criteria=self.criteria,
|
||||
scoring_type=self.scoring_type,
|
||||
threshold=self.threshold,
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
score = self._judge_output(output)
|
||||
result.evaluations.append(score)
|
||||
|
||||
if not score.passed and self.on_fail:
|
||||
self.on_fail(score)
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,184 @@
|
||||
"""
|
||||
Performance evaluator for PraisonAI Agents.
|
||||
|
||||
Measures runtime and memory usage of agent executions or functions.
|
||||
"""
|
||||
|
||||
import time
|
||||
import logging
|
||||
from typing import Callable, Optional, TYPE_CHECKING
|
||||
|
||||
from .base import BaseEvaluator
|
||||
from .results import PerformanceResult, PerformanceMetrics
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..agent.agent import Agent
|
||||
from ..agents.agents import PraisonAIAgents
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PerformanceEvaluator(BaseEvaluator):
|
||||
"""
|
||||
Evaluates the performance of agent executions or functions.
|
||||
|
||||
Measures runtime and memory usage across multiple iterations.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
func: Optional[Callable[[], None]] = None,
|
||||
agent: Optional["Agent"] = None,
|
||||
input_text: str = "",
|
||||
num_iterations: int = 10,
|
||||
warmup_runs: int = 2,
|
||||
track_memory: bool = True,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the performance evaluator.
|
||||
|
||||
Args:
|
||||
func: Callable to benchmark (takes no arguments)
|
||||
agent: Agent to benchmark (will call chat with input_text)
|
||||
input_text: Input text for agent benchmarking
|
||||
num_iterations: Number of benchmark iterations
|
||||
warmup_runs: Number of warmup runs before measurement
|
||||
track_memory: Whether to track memory usage
|
||||
name: Name for this evaluation
|
||||
save_results_path: Path to save results
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
super().__init__(name=name, save_results_path=save_results_path, verbose=verbose)
|
||||
|
||||
self.func = func
|
||||
self.agent = agent
|
||||
self.input_text = input_text
|
||||
self.num_iterations = num_iterations
|
||||
self.warmup_runs = warmup_runs
|
||||
self.track_memory = track_memory
|
||||
|
||||
if func is None and agent is None:
|
||||
raise ValueError("Either 'func' or 'agent' must be provided")
|
||||
|
||||
def _get_memory_usage(self) -> float:
|
||||
"""Get current memory usage in MB."""
|
||||
if not self.track_memory:
|
||||
return 0.0
|
||||
|
||||
try:
|
||||
import tracemalloc
|
||||
current, _ = tracemalloc.get_traced_memory()
|
||||
return current / (1024 * 1024)
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
def _execute(self) -> None:
|
||||
"""Execute the function or agent."""
|
||||
if self.func:
|
||||
self.func()
|
||||
elif self.agent:
|
||||
if hasattr(self.agent, 'chat'):
|
||||
self.agent.chat(self.input_text)
|
||||
elif hasattr(self.agent, 'start'):
|
||||
self.agent.start(self.input_text)
|
||||
else:
|
||||
raise ValueError("Agent must have 'chat' or 'start' method")
|
||||
|
||||
def run(self, print_summary: bool = False) -> PerformanceResult:
|
||||
"""
|
||||
Execute the performance evaluation.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
PerformanceResult with all performance metrics
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = PerformanceResult(
|
||||
warmup_runs=self.warmup_runs,
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
if self.track_memory:
|
||||
try:
|
||||
import tracemalloc
|
||||
tracemalloc.start()
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not start memory tracking: {e}")
|
||||
self.track_memory = False
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f"Running {self.warmup_runs} warmup iterations...")
|
||||
|
||||
for i in range(self.warmup_runs):
|
||||
try:
|
||||
self._execute()
|
||||
except Exception as e:
|
||||
logger.warning(f"Warmup iteration {i + 1} failed: {e}")
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f"Running {self.num_iterations} benchmark iterations...")
|
||||
|
||||
for i in range(self.num_iterations):
|
||||
memory_before = self._get_memory_usage()
|
||||
|
||||
start_time = time.perf_counter()
|
||||
try:
|
||||
self._execute()
|
||||
success = True
|
||||
except Exception as e:
|
||||
logger.error(f"Iteration {i + 1} failed: {e}")
|
||||
success = False
|
||||
end_time = time.perf_counter()
|
||||
|
||||
memory_after = self._get_memory_usage()
|
||||
|
||||
if success:
|
||||
metrics = PerformanceMetrics(
|
||||
run_time=end_time - start_time,
|
||||
memory_usage=max(0, memory_after - memory_before),
|
||||
iteration=i + 1
|
||||
)
|
||||
result.metrics.append(metrics)
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f" Iteration {i + 1}: {metrics.run_time:.4f}s, {metrics.memory_usage:.2f}MB")
|
||||
|
||||
if self.track_memory:
|
||||
try:
|
||||
import tracemalloc
|
||||
tracemalloc.stop()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
async def run_async(self, print_summary: bool = False) -> PerformanceResult:
|
||||
"""
|
||||
Execute the performance evaluation asynchronously.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
PerformanceResult with all performance metrics
|
||||
"""
|
||||
await self.async_before_run()
|
||||
result = self.run(print_summary=False)
|
||||
await self.async_after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,225 @@
|
||||
"""
|
||||
Reliability evaluator for PraisonAI Agents.
|
||||
|
||||
Evaluates agent reliability by verifying expected tool calls are made.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Dict, List, Optional, Set, Any, TYPE_CHECKING
|
||||
|
||||
from .base import BaseEvaluator
|
||||
from .results import ReliabilityResult, ToolCallResult
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..agent.agent import Agent
|
||||
from ..agents.agents import PraisonAIAgents
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ReliabilityEvaluator(BaseEvaluator):
|
||||
"""
|
||||
Evaluates the reliability of agent tool usage.
|
||||
|
||||
Verifies that expected tools are called during agent execution.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
agent: Optional["Agent"] = None,
|
||||
input_text: str = "",
|
||||
expected_tools: Optional[List[str]] = None,
|
||||
forbidden_tools: Optional[List[str]] = None,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the reliability evaluator.
|
||||
|
||||
Args:
|
||||
agent: Agent to evaluate
|
||||
input_text: Input to provide to the agent
|
||||
expected_tools: List of tool names that should be called
|
||||
forbidden_tools: List of tool names that should NOT be called
|
||||
name: Name for this evaluation
|
||||
save_results_path: Path to save results
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
super().__init__(name=name, save_results_path=save_results_path, verbose=verbose)
|
||||
|
||||
self.agent = agent
|
||||
self.input_text = input_text
|
||||
self.expected_tools = set(expected_tools or [])
|
||||
self.forbidden_tools = set(forbidden_tools or [])
|
||||
|
||||
if agent is None:
|
||||
raise ValueError("'agent' must be provided")
|
||||
|
||||
def _extract_tool_calls(self, response: Any) -> Set[str]:
|
||||
"""
|
||||
Extract tool names from agent response.
|
||||
|
||||
Args:
|
||||
response: Agent response object
|
||||
|
||||
Returns:
|
||||
Set of tool names that were called
|
||||
"""
|
||||
tool_calls = set()
|
||||
|
||||
if hasattr(response, 'tool_calls') and response.tool_calls:
|
||||
for tc in response.tool_calls:
|
||||
if hasattr(tc, 'name'):
|
||||
tool_calls.add(tc.name)
|
||||
elif hasattr(tc, 'function') and hasattr(tc.function, 'name'):
|
||||
tool_calls.add(tc.function.name)
|
||||
|
||||
if hasattr(self.agent, 'chat_history'):
|
||||
for msg in self.agent.chat_history:
|
||||
if isinstance(msg, dict):
|
||||
if msg.get('role') == 'assistant' and 'tool_calls' in msg:
|
||||
for tc in msg.get('tool_calls', []):
|
||||
if isinstance(tc, dict) and 'function' in tc:
|
||||
tool_calls.add(tc['function'].get('name', ''))
|
||||
elif msg.get('role') == 'tool':
|
||||
tool_name = msg.get('name', '')
|
||||
if tool_name:
|
||||
tool_calls.add(tool_name)
|
||||
|
||||
return tool_calls
|
||||
|
||||
def run(self, print_summary: bool = False) -> ReliabilityResult:
|
||||
"""
|
||||
Execute the reliability evaluation.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
ReliabilityResult with tool call verification results
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = ReliabilityResult(
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
try:
|
||||
if hasattr(self.agent, 'chat'):
|
||||
response = self.agent.chat(self.input_text)
|
||||
elif hasattr(self.agent, 'start'):
|
||||
response = self.agent.start(self.input_text)
|
||||
else:
|
||||
raise ValueError("Agent must have 'chat' or 'start' method")
|
||||
|
||||
actual_tools = self._extract_tool_calls(response)
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f"Expected tools: {self.expected_tools}")
|
||||
logger.info(f"Actual tools called: {actual_tools}")
|
||||
|
||||
for tool in self.expected_tools:
|
||||
was_called = tool in actual_tools
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=True,
|
||||
actual=was_called,
|
||||
passed=was_called
|
||||
))
|
||||
|
||||
for tool in self.forbidden_tools:
|
||||
was_called = tool in actual_tools
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=False,
|
||||
actual=was_called,
|
||||
passed=not was_called
|
||||
))
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during reliability evaluation: {e}")
|
||||
for tool in self.expected_tools:
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=True,
|
||||
actual=False,
|
||||
passed=False,
|
||||
metadata={"error": str(e)}
|
||||
))
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
async def run_async(self, print_summary: bool = False) -> ReliabilityResult:
|
||||
"""
|
||||
Execute the reliability evaluation asynchronously.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
ReliabilityResult with tool call verification results
|
||||
"""
|
||||
await self.async_before_run()
|
||||
result = self.run(print_summary=False)
|
||||
await self.async_after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
def evaluate_tool_calls(
|
||||
self,
|
||||
actual_tools: List[str],
|
||||
print_summary: bool = False
|
||||
) -> ReliabilityResult:
|
||||
"""
|
||||
Evaluate a pre-recorded list of tool calls without running the agent.
|
||||
|
||||
Args:
|
||||
actual_tools: List of tool names that were called
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
ReliabilityResult with tool call verification results
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = ReliabilityResult(
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
actual_set = set(actual_tools)
|
||||
|
||||
for tool in self.expected_tools:
|
||||
was_called = tool in actual_set
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=True,
|
||||
actual=was_called,
|
||||
passed=was_called
|
||||
))
|
||||
|
||||
for tool in self.forbidden_tools:
|
||||
was_called = tool in actual_set
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=False,
|
||||
actual=was_called,
|
||||
passed=not was_called
|
||||
))
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,463 @@
|
||||
"""
|
||||
Result dataclasses for PraisonAI Agents evaluation framework.
|
||||
|
||||
This module provides structured result types for different evaluation methods.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from typing import List, Optional, Dict, Any
|
||||
from datetime import datetime
|
||||
import statistics
|
||||
import json
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvaluationScore:
|
||||
"""Individual evaluation score with reasoning."""
|
||||
score: float
|
||||
reasoning: str
|
||||
input_text: str
|
||||
output_text: str
|
||||
expected_output: Optional[str] = None
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AccuracyResult:
|
||||
"""Result from accuracy evaluation comparing output to expected output."""
|
||||
evaluations: List[EvaluationScore] = field(default_factory=list)
|
||||
eval_id: str = ""
|
||||
name: str = ""
|
||||
|
||||
@property
|
||||
def scores(self) -> List[float]:
|
||||
"""Get list of all scores."""
|
||||
return [e.score for e in self.evaluations]
|
||||
|
||||
@property
|
||||
def avg_score(self) -> float:
|
||||
"""Calculate average score."""
|
||||
if not self.scores:
|
||||
return 0.0
|
||||
return statistics.mean(self.scores)
|
||||
|
||||
@property
|
||||
def min_score(self) -> float:
|
||||
"""Get minimum score."""
|
||||
if not self.scores:
|
||||
return 0.0
|
||||
return min(self.scores)
|
||||
|
||||
@property
|
||||
def max_score(self) -> float:
|
||||
"""Get maximum score."""
|
||||
if not self.scores:
|
||||
return 0.0
|
||||
return max(self.scores)
|
||||
|
||||
@property
|
||||
def std_dev(self) -> float:
|
||||
"""Calculate standard deviation of scores."""
|
||||
if len(self.scores) < 2:
|
||||
return 0.0
|
||||
return statistics.stdev(self.scores)
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
"""Check if evaluation passed (avg score >= 7)."""
|
||||
return self.avg_score >= 7.0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary."""
|
||||
return {
|
||||
"eval_id": self.eval_id,
|
||||
"name": self.name,
|
||||
"evaluations": [asdict(e) for e in self.evaluations],
|
||||
"avg_score": self.avg_score,
|
||||
"min_score": self.min_score,
|
||||
"max_score": self.max_score,
|
||||
"std_dev": self.std_dev,
|
||||
"passed": self.passed,
|
||||
"num_evaluations": len(self.evaluations)
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Convert result to JSON string."""
|
||||
return json.dumps(self.to_dict(), indent=2)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print a summary of the evaluation results."""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
from rich.panel import Panel # noqa: F401
|
||||
|
||||
console = Console()
|
||||
|
||||
table = Table(title="Accuracy Evaluation Summary")
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Evaluations", str(len(self.evaluations)))
|
||||
table.add_row("Average Score", f"{self.avg_score:.2f}")
|
||||
table.add_row("Min Score", f"{self.min_score:.2f}")
|
||||
table.add_row("Max Score", f"{self.max_score:.2f}")
|
||||
table.add_row("Std Dev", f"{self.std_dev:.2f}")
|
||||
table.add_row("Status", "✅ PASSED" if self.passed else "❌ FAILED")
|
||||
|
||||
console.print(table)
|
||||
except ImportError:
|
||||
print("Accuracy Evaluation Summary")
|
||||
print(f" Evaluations: {len(self.evaluations)}")
|
||||
print(f" Average Score: {self.avg_score:.2f}")
|
||||
print(f" Min Score: {self.min_score:.2f}")
|
||||
print(f" Max Score: {self.max_score:.2f}")
|
||||
print(f" Std Dev: {self.std_dev:.2f}")
|
||||
print(f" Status: {'PASSED' if self.passed else 'FAILED'}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class PerformanceMetrics:
|
||||
"""Individual performance measurement."""
|
||||
run_time: float
|
||||
memory_usage: float
|
||||
iteration: int
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PerformanceResult:
|
||||
"""Result from performance evaluation measuring runtime and memory."""
|
||||
metrics: List[PerformanceMetrics] = field(default_factory=list)
|
||||
warmup_runs: int = 0
|
||||
eval_id: str = ""
|
||||
name: str = ""
|
||||
|
||||
@property
|
||||
def run_times(self) -> List[float]:
|
||||
"""Get list of all run times."""
|
||||
return [m.run_time for m in self.metrics]
|
||||
|
||||
@property
|
||||
def memory_usages(self) -> List[float]:
|
||||
"""Get list of all memory usages."""
|
||||
return [m.memory_usage for m in self.metrics]
|
||||
|
||||
@property
|
||||
def avg_run_time(self) -> float:
|
||||
"""Calculate average run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
return statistics.mean(self.run_times)
|
||||
|
||||
@property
|
||||
def min_run_time(self) -> float:
|
||||
"""Get minimum run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
return min(self.run_times)
|
||||
|
||||
@property
|
||||
def max_run_time(self) -> float:
|
||||
"""Get maximum run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
return max(self.run_times)
|
||||
|
||||
@property
|
||||
def std_dev_run_time(self) -> float:
|
||||
"""Calculate standard deviation of run times."""
|
||||
if len(self.run_times) < 2:
|
||||
return 0.0
|
||||
return statistics.stdev(self.run_times)
|
||||
|
||||
@property
|
||||
def median_run_time(self) -> float:
|
||||
"""Calculate median run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
return statistics.median(self.run_times)
|
||||
|
||||
@property
|
||||
def p95_run_time(self) -> float:
|
||||
"""Calculate 95th percentile run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
sorted_times = sorted(self.run_times)
|
||||
idx = int(len(sorted_times) * 0.95)
|
||||
return sorted_times[min(idx, len(sorted_times) - 1)]
|
||||
|
||||
@property
|
||||
def avg_memory(self) -> float:
|
||||
"""Calculate average memory usage."""
|
||||
if not self.memory_usages:
|
||||
return 0.0
|
||||
return statistics.mean(self.memory_usages)
|
||||
|
||||
@property
|
||||
def max_memory(self) -> float:
|
||||
"""Get maximum memory usage."""
|
||||
if not self.memory_usages:
|
||||
return 0.0
|
||||
return max(self.memory_usages)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary."""
|
||||
return {
|
||||
"eval_id": self.eval_id,
|
||||
"name": self.name,
|
||||
"metrics": [asdict(m) for m in self.metrics],
|
||||
"warmup_runs": self.warmup_runs,
|
||||
"avg_run_time": self.avg_run_time,
|
||||
"min_run_time": self.min_run_time,
|
||||
"max_run_time": self.max_run_time,
|
||||
"std_dev_run_time": self.std_dev_run_time,
|
||||
"median_run_time": self.median_run_time,
|
||||
"p95_run_time": self.p95_run_time,
|
||||
"avg_memory": self.avg_memory,
|
||||
"max_memory": self.max_memory,
|
||||
"num_iterations": len(self.metrics)
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Convert result to JSON string."""
|
||||
return json.dumps(self.to_dict(), indent=2)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print a summary of the performance results."""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
|
||||
table = Table(title="Performance Evaluation Summary")
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Iterations", str(len(self.metrics)))
|
||||
table.add_row("Warmup Runs", str(self.warmup_runs))
|
||||
table.add_row("Avg Run Time", f"{self.avg_run_time:.4f}s")
|
||||
table.add_row("Min Run Time", f"{self.min_run_time:.4f}s")
|
||||
table.add_row("Max Run Time", f"{self.max_run_time:.4f}s")
|
||||
table.add_row("Std Dev", f"{self.std_dev_run_time:.4f}s")
|
||||
table.add_row("Median", f"{self.median_run_time:.4f}s")
|
||||
table.add_row("P95", f"{self.p95_run_time:.4f}s")
|
||||
table.add_row("Avg Memory", f"{self.avg_memory:.2f} MB")
|
||||
table.add_row("Max Memory", f"{self.max_memory:.2f} MB")
|
||||
|
||||
console.print(table)
|
||||
except ImportError:
|
||||
print("Performance Evaluation Summary")
|
||||
print(f" Iterations: {len(self.metrics)}")
|
||||
print(f" Warmup Runs: {self.warmup_runs}")
|
||||
print(f" Avg Run Time: {self.avg_run_time:.4f}s")
|
||||
print(f" Min Run Time: {self.min_run_time:.4f}s")
|
||||
print(f" Max Run Time: {self.max_run_time:.4f}s")
|
||||
print(f" Std Dev: {self.std_dev_run_time:.4f}s")
|
||||
print(f" Median: {self.median_run_time:.4f}s")
|
||||
print(f" P95: {self.p95_run_time:.4f}s")
|
||||
print(f" Avg Memory: {self.avg_memory:.2f} MB")
|
||||
print(f" Max Memory: {self.max_memory:.2f} MB")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCallResult:
|
||||
"""Result of a single tool call verification."""
|
||||
tool_name: str
|
||||
expected: bool
|
||||
actual: bool
|
||||
passed: bool
|
||||
arguments: Optional[Dict[str, Any]] = None
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReliabilityResult:
|
||||
"""Result from reliability evaluation checking tool calls."""
|
||||
tool_results: List[ToolCallResult] = field(default_factory=list)
|
||||
eval_id: str = ""
|
||||
name: str = ""
|
||||
|
||||
@property
|
||||
def passed_calls(self) -> List[ToolCallResult]:
|
||||
"""Get list of passed tool calls."""
|
||||
return [t for t in self.tool_results if t.passed]
|
||||
|
||||
@property
|
||||
def failed_calls(self) -> List[ToolCallResult]:
|
||||
"""Get list of failed tool calls."""
|
||||
return [t for t in self.tool_results if not t.passed]
|
||||
|
||||
@property
|
||||
def pass_rate(self) -> float:
|
||||
"""Calculate pass rate."""
|
||||
if not self.tool_results:
|
||||
return 0.0
|
||||
return len(self.passed_calls) / len(self.tool_results)
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
"""Get overall status."""
|
||||
return "PASSED" if len(self.failed_calls) == 0 else "FAILED"
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
"""Check if all tool calls passed."""
|
||||
return len(self.failed_calls) == 0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary."""
|
||||
return {
|
||||
"eval_id": self.eval_id,
|
||||
"name": self.name,
|
||||
"tool_results": [asdict(t) for t in self.tool_results],
|
||||
"passed_count": len(self.passed_calls),
|
||||
"failed_count": len(self.failed_calls),
|
||||
"pass_rate": self.pass_rate,
|
||||
"status": self.status
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Convert result to JSON string."""
|
||||
return json.dumps(self.to_dict(), indent=2)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print a summary of the reliability results."""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
|
||||
table = Table(title="Reliability Evaluation Summary")
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Total Tool Calls", str(len(self.tool_results)))
|
||||
table.add_row("Passed", str(len(self.passed_calls)))
|
||||
table.add_row("Failed", str(len(self.failed_calls)))
|
||||
table.add_row("Pass Rate", f"{self.pass_rate:.1%}")
|
||||
table.add_row("Status", "✅ PASSED" if self.passed else "❌ FAILED")
|
||||
|
||||
console.print(table)
|
||||
|
||||
if self.failed_calls:
|
||||
fail_table = Table(title="Failed Tool Calls")
|
||||
fail_table.add_column("Tool", style="red")
|
||||
fail_table.add_column("Expected", style="yellow")
|
||||
fail_table.add_column("Actual", style="yellow")
|
||||
|
||||
for tc in self.failed_calls:
|
||||
fail_table.add_row(tc.tool_name, str(tc.expected), str(tc.actual))
|
||||
|
||||
console.print(fail_table)
|
||||
except ImportError:
|
||||
print("Reliability Evaluation Summary")
|
||||
print(f" Total Tool Calls: {len(self.tool_results)}")
|
||||
print(f" Passed: {len(self.passed_calls)}")
|
||||
print(f" Failed: {len(self.failed_calls)}")
|
||||
print(f" Pass Rate: {self.pass_rate:.1%}")
|
||||
print(f" Status: {self.status}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class CriteriaScore:
|
||||
"""Individual criteria evaluation score."""
|
||||
score: float
|
||||
passed: bool
|
||||
reasoning: str
|
||||
output_text: str
|
||||
criteria: str
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CriteriaResult:
|
||||
"""Result from criteria-based evaluation (agent-as-judge)."""
|
||||
evaluations: List[CriteriaScore] = field(default_factory=list)
|
||||
criteria: str = ""
|
||||
scoring_type: str = "numeric" # "numeric" or "binary"
|
||||
threshold: float = 7.0
|
||||
eval_id: str = ""
|
||||
name: str = ""
|
||||
|
||||
@property
|
||||
def scores(self) -> List[float]:
|
||||
"""Get list of all scores."""
|
||||
return [e.score for e in self.evaluations]
|
||||
|
||||
@property
|
||||
def avg_score(self) -> float:
|
||||
"""Calculate average score."""
|
||||
if not self.scores:
|
||||
return 0.0
|
||||
return statistics.mean(self.scores)
|
||||
|
||||
@property
|
||||
def pass_rate(self) -> float:
|
||||
"""Calculate pass rate."""
|
||||
if not self.evaluations:
|
||||
return 0.0
|
||||
passed = sum(1 for e in self.evaluations if e.passed)
|
||||
return passed / len(self.evaluations)
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
"""Check if evaluation passed overall."""
|
||||
if self.scoring_type == "binary":
|
||||
return self.pass_rate >= 0.5
|
||||
return self.avg_score >= self.threshold
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary."""
|
||||
return {
|
||||
"eval_id": self.eval_id,
|
||||
"name": self.name,
|
||||
"criteria": self.criteria,
|
||||
"scoring_type": self.scoring_type,
|
||||
"threshold": self.threshold,
|
||||
"evaluations": [asdict(e) for e in self.evaluations],
|
||||
"avg_score": self.avg_score,
|
||||
"pass_rate": self.pass_rate,
|
||||
"passed": self.passed,
|
||||
"num_evaluations": len(self.evaluations)
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Convert result to JSON string."""
|
||||
return json.dumps(self.to_dict(), indent=2)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print a summary of the criteria evaluation results."""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
|
||||
table = Table(title="Criteria Evaluation Summary")
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Criteria", self.criteria[:50] + "..." if len(self.criteria) > 50 else self.criteria)
|
||||
table.add_row("Scoring Type", self.scoring_type)
|
||||
table.add_row("Threshold", str(self.threshold))
|
||||
table.add_row("Evaluations", str(len(self.evaluations)))
|
||||
table.add_row("Average Score", f"{self.avg_score:.2f}")
|
||||
table.add_row("Pass Rate", f"{self.pass_rate:.1%}")
|
||||
table.add_row("Status", "✅ PASSED" if self.passed else "❌ FAILED")
|
||||
|
||||
console.print(table)
|
||||
except ImportError:
|
||||
print("Criteria Evaluation Summary")
|
||||
print(f" Criteria: {self.criteria[:50]}...")
|
||||
print(f" Scoring Type: {self.scoring_type}")
|
||||
print(f" Threshold: {self.threshold}")
|
||||
print(f" Evaluations: {len(self.evaluations)}")
|
||||
print(f" Average Score: {self.avg_score:.2f}")
|
||||
print(f" Pass Rate: {self.pass_rate:.1%}")
|
||||
print(f" Status: {'PASSED' if self.passed else 'FAILED'}")
|
||||
@@ -0,0 +1,154 @@
|
||||
"""
|
||||
Utility functions for PraisonAI Agents evaluation framework.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Union
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def save_result_to_file(
|
||||
result: Any,
|
||||
file_path: str,
|
||||
eval_id: Optional[str] = None,
|
||||
name: Optional[str] = None
|
||||
) -> bool:
|
||||
"""
|
||||
Save evaluation result to a file.
|
||||
|
||||
Args:
|
||||
result: The evaluation result to save
|
||||
file_path: Path to save the result (supports {name}, {eval_id} placeholders)
|
||||
eval_id: Optional eval ID for placeholder substitution
|
||||
name: Optional name for placeholder substitution
|
||||
|
||||
Returns:
|
||||
True if save was successful, False otherwise
|
||||
"""
|
||||
try:
|
||||
formatted_path = file_path.format(name=name or "", eval_id=eval_id or "")
|
||||
path = Path(formatted_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if hasattr(result, 'to_dict'):
|
||||
data = result.to_dict()
|
||||
elif hasattr(result, '__dict__'):
|
||||
data = result.__dict__
|
||||
else:
|
||||
data = {"result": str(result)}
|
||||
|
||||
path.write_text(json.dumps(data, indent=2, default=str))
|
||||
logger.debug(f"Saved result to: {formatted_path}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save result: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def load_result_from_file(file_path: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Load evaluation result from a file.
|
||||
|
||||
Args:
|
||||
file_path: Path to the result file
|
||||
|
||||
Returns:
|
||||
Dictionary containing the result data, or None if load failed
|
||||
"""
|
||||
try:
|
||||
path = Path(file_path)
|
||||
if not path.exists():
|
||||
logger.warning(f"Result file not found: {file_path}")
|
||||
return None
|
||||
|
||||
data = json.loads(path.read_text())
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load result: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def format_score(score: float, max_score: float = 10.0) -> str:
|
||||
"""
|
||||
Format a score for display.
|
||||
|
||||
Args:
|
||||
score: The score value
|
||||
max_score: Maximum possible score
|
||||
|
||||
Returns:
|
||||
Formatted score string
|
||||
"""
|
||||
return f"{score:.1f}/{max_score:.0f}"
|
||||
|
||||
|
||||
def format_percentage(value: float) -> str:
|
||||
"""
|
||||
Format a value as a percentage.
|
||||
|
||||
Args:
|
||||
value: Value between 0 and 1
|
||||
|
||||
Returns:
|
||||
Formatted percentage string
|
||||
"""
|
||||
return f"{value * 100:.1f}%"
|
||||
|
||||
|
||||
def format_duration(seconds: float) -> str:
|
||||
"""
|
||||
Format a duration in seconds for display.
|
||||
|
||||
Args:
|
||||
seconds: Duration in seconds
|
||||
|
||||
Returns:
|
||||
Formatted duration string
|
||||
"""
|
||||
if seconds < 0.001:
|
||||
return f"{seconds * 1000000:.0f}μs"
|
||||
elif seconds < 1:
|
||||
return f"{seconds * 1000:.2f}ms"
|
||||
elif seconds < 60:
|
||||
return f"{seconds:.2f}s"
|
||||
else:
|
||||
minutes = int(seconds // 60)
|
||||
secs = seconds % 60
|
||||
return f"{minutes}m {secs:.1f}s"
|
||||
|
||||
|
||||
def format_memory(mb: float) -> str:
|
||||
"""
|
||||
Format memory usage for display.
|
||||
|
||||
Args:
|
||||
mb: Memory in megabytes
|
||||
|
||||
Returns:
|
||||
Formatted memory string
|
||||
"""
|
||||
if mb < 1:
|
||||
return f"{mb * 1024:.1f} KB"
|
||||
elif mb < 1024:
|
||||
return f"{mb:.2f} MB"
|
||||
else:
|
||||
return f"{mb / 1024:.2f} GB"
|
||||
|
||||
|
||||
def calculate_pass_rate(passed: int, total: int) -> float:
|
||||
"""
|
||||
Calculate pass rate.
|
||||
|
||||
Args:
|
||||
passed: Number of passed evaluations
|
||||
total: Total number of evaluations
|
||||
|
||||
Returns:
|
||||
Pass rate as a float between 0 and 1
|
||||
"""
|
||||
if total == 0:
|
||||
return 0.0
|
||||
return passed / total
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Agent Skills module for PraisonAI Agents.
|
||||
|
||||
This module provides support for the open Agent Skills standard (agentskills.io),
|
||||
enabling agents to load and use modular capabilities through SKILL.md files.
|
||||
|
||||
Zero Performance Impact:
|
||||
- All imports are lazy loaded via __getattr__
|
||||
- Skills only loaded when explicitly enabled
|
||||
- No auto-discovery at import time
|
||||
|
||||
Usage:
|
||||
from praisonaiagents.skills import SkillManager, SkillProperties
|
||||
|
||||
manager = SkillManager()
|
||||
manager.discover(["./skills"])
|
||||
prompt_xml = manager.to_prompt()
|
||||
"""
|
||||
|
||||
__all__ = [
|
||||
# Models
|
||||
"SkillProperties",
|
||||
"SkillMetadata",
|
||||
"ParseError",
|
||||
"ValidationError",
|
||||
# Parser
|
||||
"parse_frontmatter",
|
||||
"find_skill_md",
|
||||
"read_properties",
|
||||
# Validator
|
||||
"validate",
|
||||
"validate_metadata",
|
||||
# Prompt
|
||||
"to_prompt",
|
||||
"generate_skills_xml",
|
||||
"format_skill_for_prompt",
|
||||
# Discovery
|
||||
"discover_skills",
|
||||
"get_default_skill_dirs",
|
||||
# Loader
|
||||
"SkillLoader",
|
||||
# Manager
|
||||
"SkillManager",
|
||||
]
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
"""Lazy load module components to avoid import overhead."""
|
||||
if name in ("SkillProperties", "SkillMetadata", "ParseError", "ValidationError"):
|
||||
from .models import SkillProperties, SkillMetadata, ParseError, ValidationError
|
||||
return locals()[name]
|
||||
|
||||
if name in ("parse_frontmatter", "find_skill_md", "read_properties"):
|
||||
from .parser import parse_frontmatter, find_skill_md, read_properties
|
||||
return locals()[name]
|
||||
|
||||
if name in ("validate", "validate_metadata", "_validate_name", "_validate_description", "_validate_compatibility"):
|
||||
from .validator import validate, validate_metadata, _validate_name, _validate_description, _validate_compatibility
|
||||
return locals()[name]
|
||||
|
||||
if name in ("to_prompt", "generate_skills_xml", "format_skill_for_prompt"):
|
||||
from .prompt import to_prompt, generate_skills_xml, format_skill_for_prompt
|
||||
return locals()[name]
|
||||
|
||||
if name in ("discover_skills", "get_default_skill_dirs"):
|
||||
from .discovery import discover_skills, get_default_skill_dirs
|
||||
return locals()[name]
|
||||
|
||||
if name == "SkillLoader":
|
||||
from .loader import SkillLoader
|
||||
return SkillLoader
|
||||
|
||||
if name == "SkillManager":
|
||||
from .manager import SkillManager
|
||||
return SkillManager
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Skill discovery and directory scanning."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
from .parser import find_skill_md, read_properties
|
||||
from .models import SkillProperties
|
||||
|
||||
|
||||
def get_default_skill_dirs() -> List[Path]:
|
||||
"""Get default skill directory locations.
|
||||
|
||||
Returns directories in precedence order (high to low):
|
||||
1. Project: ./.praison/skills/ or ./.claude/skills/
|
||||
2. User: ~/.praison/skills/
|
||||
3. System: /etc/praison/skills/ (admin-managed)
|
||||
|
||||
Returns:
|
||||
List of existing skill directories
|
||||
"""
|
||||
dirs = []
|
||||
cwd = Path.cwd()
|
||||
|
||||
# Project-level directories
|
||||
project_dirs = [
|
||||
cwd / ".praison" / "skills",
|
||||
cwd / ".claude" / "skills",
|
||||
]
|
||||
for d in project_dirs:
|
||||
if d.exists() and d.is_dir():
|
||||
dirs.append(d)
|
||||
|
||||
# User-level directory
|
||||
user_dir = Path.home() / ".praison" / "skills"
|
||||
if user_dir.exists() and user_dir.is_dir():
|
||||
dirs.append(user_dir)
|
||||
|
||||
# System-level directory (Unix-like systems)
|
||||
system_dir = Path("/etc/praison/skills")
|
||||
if system_dir.exists() and system_dir.is_dir():
|
||||
dirs.append(system_dir)
|
||||
|
||||
return dirs
|
||||
|
||||
|
||||
def discover_skills(
|
||||
skill_dirs: Optional[List[str]] = None,
|
||||
include_defaults: bool = True,
|
||||
) -> List[SkillProperties]:
|
||||
"""Discover all valid skills in the given directories.
|
||||
|
||||
Args:
|
||||
skill_dirs: List of directory paths to scan for skills.
|
||||
Each directory should contain skill subdirectories.
|
||||
include_defaults: Whether to include default skill directories
|
||||
|
||||
Returns:
|
||||
List of SkillProperties for all valid skills found
|
||||
"""
|
||||
all_dirs = []
|
||||
|
||||
# Add explicit directories
|
||||
if skill_dirs:
|
||||
for d in skill_dirs:
|
||||
path = Path(d).expanduser().resolve()
|
||||
if path.exists() and path.is_dir():
|
||||
all_dirs.append(path)
|
||||
|
||||
# Add default directories
|
||||
if include_defaults:
|
||||
all_dirs.extend(get_default_skill_dirs())
|
||||
|
||||
# Remove duplicates while preserving order
|
||||
seen = set()
|
||||
unique_dirs = []
|
||||
for d in all_dirs:
|
||||
if d not in seen:
|
||||
seen.add(d)
|
||||
unique_dirs.append(d)
|
||||
|
||||
skills = []
|
||||
|
||||
for parent_dir in unique_dirs:
|
||||
# Each subdirectory in parent_dir might be a skill
|
||||
try:
|
||||
for item in parent_dir.iterdir():
|
||||
if not item.is_dir():
|
||||
continue
|
||||
|
||||
# Check if this directory contains a SKILL.md
|
||||
skill_md = find_skill_md(item)
|
||||
if skill_md is None:
|
||||
continue
|
||||
|
||||
try:
|
||||
props = read_properties(item)
|
||||
skills.append(props)
|
||||
except Exception:
|
||||
# Skip invalid skills
|
||||
continue
|
||||
except PermissionError:
|
||||
# Skip directories we can't read
|
||||
continue
|
||||
|
||||
return skills
|
||||
|
||||
|
||||
def discover_skill(skill_path: str) -> Optional[SkillProperties]:
|
||||
"""Discover a single skill from a directory path.
|
||||
|
||||
Args:
|
||||
skill_path: Path to a skill directory
|
||||
|
||||
Returns:
|
||||
SkillProperties if valid, None otherwise
|
||||
"""
|
||||
path = Path(skill_path).expanduser().resolve()
|
||||
|
||||
if not path.exists() or not path.is_dir():
|
||||
return None
|
||||
|
||||
skill_md = find_skill_md(path)
|
||||
if skill_md is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
return read_properties(path)
|
||||
except Exception:
|
||||
return None
|
||||
@@ -0,0 +1,236 @@
|
||||
"""Progressive skill loading for Agent Skills."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .parser import find_skill_md, read_properties, parse_frontmatter
|
||||
from .models import SkillProperties, SkillMetadata
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoadedSkill:
|
||||
"""A fully or partially loaded skill.
|
||||
|
||||
Supports progressive disclosure:
|
||||
- Level 1: metadata only (name, description, location)
|
||||
- Level 2: full instructions (SKILL.md body)
|
||||
- Level 3: resources (scripts, references, assets)
|
||||
"""
|
||||
|
||||
properties: SkillProperties
|
||||
instructions: Optional[str] = None
|
||||
resources_loaded: bool = False
|
||||
_scripts: dict = field(default_factory=dict)
|
||||
_references: dict = field(default_factory=dict)
|
||||
_assets: dict = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def metadata(self) -> SkillMetadata:
|
||||
"""Get lightweight metadata for system prompt."""
|
||||
return SkillMetadata.from_properties(self.properties)
|
||||
|
||||
@property
|
||||
def is_activated(self) -> bool:
|
||||
"""Check if skill has been activated (instructions loaded)."""
|
||||
return self.instructions is not None
|
||||
|
||||
def get_scripts(self) -> dict:
|
||||
"""Get loaded scripts."""
|
||||
return self._scripts
|
||||
|
||||
def get_references(self) -> dict:
|
||||
"""Get loaded references."""
|
||||
return self._references
|
||||
|
||||
def get_assets(self) -> dict:
|
||||
"""Get loaded assets."""
|
||||
return self._assets
|
||||
|
||||
|
||||
class SkillLoader:
|
||||
"""Loader for Agent Skills with progressive disclosure support.
|
||||
|
||||
Usage:
|
||||
loader = SkillLoader()
|
||||
|
||||
# Level 1: Load metadata only
|
||||
skill = loader.load_metadata("/path/to/skill")
|
||||
|
||||
# Level 2: Activate skill (load instructions)
|
||||
loader.activate(skill)
|
||||
|
||||
# Level 3: Load specific resources
|
||||
loader.load_scripts(skill)
|
||||
loader.load_references(skill)
|
||||
"""
|
||||
|
||||
def load_metadata(self, skill_path: str) -> Optional[LoadedSkill]:
|
||||
"""Load skill metadata only (Level 1).
|
||||
|
||||
This is the minimal load for system prompt injection.
|
||||
Only reads the YAML frontmatter, not the full SKILL.md body.
|
||||
|
||||
Args:
|
||||
skill_path: Path to skill directory
|
||||
|
||||
Returns:
|
||||
LoadedSkill with metadata only, or None if invalid
|
||||
"""
|
||||
path = Path(skill_path).expanduser().resolve()
|
||||
|
||||
if not path.exists() or not path.is_dir():
|
||||
return None
|
||||
|
||||
try:
|
||||
props = read_properties(path)
|
||||
return LoadedSkill(properties=props)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def activate(self, skill: LoadedSkill) -> bool:
|
||||
"""Activate a skill by loading its full instructions (Level 2).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance to activate
|
||||
|
||||
Returns:
|
||||
True if activation succeeded
|
||||
"""
|
||||
if skill.is_activated:
|
||||
return True
|
||||
|
||||
if skill.properties.path is None:
|
||||
return False
|
||||
|
||||
skill_md = find_skill_md(skill.properties.path)
|
||||
if skill_md is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
content = skill_md.read_text()
|
||||
_, body = parse_frontmatter(content)
|
||||
skill.instructions = body
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def load_scripts(self, skill: LoadedSkill) -> dict:
|
||||
"""Load scripts from skill's scripts/ directory (Level 3).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance
|
||||
|
||||
Returns:
|
||||
Dict mapping script names to their contents
|
||||
"""
|
||||
if skill.properties.path is None:
|
||||
return {}
|
||||
|
||||
scripts_dir = skill.properties.path / "scripts"
|
||||
if not scripts_dir.exists() or not scripts_dir.is_dir():
|
||||
return {}
|
||||
|
||||
scripts = {}
|
||||
try:
|
||||
for script_file in scripts_dir.iterdir():
|
||||
if script_file.is_file():
|
||||
try:
|
||||
scripts[script_file.name] = script_file.read_text()
|
||||
except Exception:
|
||||
continue
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
skill._scripts = scripts
|
||||
return scripts
|
||||
|
||||
def load_references(self, skill: LoadedSkill) -> dict:
|
||||
"""Load references from skill's references/ directory (Level 3).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance
|
||||
|
||||
Returns:
|
||||
Dict mapping reference names to their contents
|
||||
"""
|
||||
if skill.properties.path is None:
|
||||
return {}
|
||||
|
||||
refs_dir = skill.properties.path / "references"
|
||||
if not refs_dir.exists() or not refs_dir.is_dir():
|
||||
return {}
|
||||
|
||||
refs = {}
|
||||
try:
|
||||
for ref_file in refs_dir.iterdir():
|
||||
if ref_file.is_file():
|
||||
try:
|
||||
refs[ref_file.name] = ref_file.read_text()
|
||||
except Exception:
|
||||
continue
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
skill._references = refs
|
||||
return refs
|
||||
|
||||
def load_assets(self, skill: LoadedSkill) -> dict:
|
||||
"""Load asset paths from skill's assets/ directory (Level 3).
|
||||
|
||||
Note: For binary assets, only paths are returned, not contents.
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance
|
||||
|
||||
Returns:
|
||||
Dict mapping asset names to their paths
|
||||
"""
|
||||
if skill.properties.path is None:
|
||||
return {}
|
||||
|
||||
assets_dir = skill.properties.path / "assets"
|
||||
if not assets_dir.exists() or not assets_dir.is_dir():
|
||||
return {}
|
||||
|
||||
assets = {}
|
||||
try:
|
||||
for asset_file in assets_dir.iterdir():
|
||||
if asset_file.is_file():
|
||||
assets[asset_file.name] = str(asset_file)
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
skill._assets = assets
|
||||
skill.resources_loaded = True
|
||||
return assets
|
||||
|
||||
def load_all_resources(self, skill: LoadedSkill) -> None:
|
||||
"""Load all resources for a skill (Level 3 complete).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance
|
||||
"""
|
||||
self.load_scripts(skill)
|
||||
self.load_references(skill)
|
||||
self.load_assets(skill)
|
||||
skill.resources_loaded = True
|
||||
|
||||
@classmethod
|
||||
def load(cls, skill_path: str, activate: bool = False) -> Optional[LoadedSkill]:
|
||||
"""Convenience method to load a skill.
|
||||
|
||||
Args:
|
||||
skill_path: Path to skill directory
|
||||
activate: Whether to also load instructions
|
||||
|
||||
Returns:
|
||||
LoadedSkill or None if invalid
|
||||
"""
|
||||
loader = cls()
|
||||
skill = loader.load_metadata(skill_path)
|
||||
|
||||
if skill and activate:
|
||||
loader.activate(skill)
|
||||
|
||||
return skill
|
||||
@@ -0,0 +1,189 @@
|
||||
"""SkillManager for Agent Skills integration."""
|
||||
|
||||
from typing import List, Optional, Dict
|
||||
|
||||
from .models import SkillMetadata
|
||||
from .discovery import discover_skills
|
||||
from .loader import SkillLoader, LoadedSkill
|
||||
from .prompt import generate_skills_xml
|
||||
|
||||
|
||||
class SkillManager:
|
||||
"""Manager for discovering, loading, and using Agent Skills.
|
||||
|
||||
This is the main entry point for Agent Skills integration.
|
||||
Supports progressive disclosure and zero performance impact
|
||||
when skills are not used.
|
||||
|
||||
Usage:
|
||||
manager = SkillManager()
|
||||
|
||||
# Discover skills from directories
|
||||
manager.discover(["./skills", "~/.praison/skills"])
|
||||
|
||||
# Get XML for system prompt
|
||||
prompt_xml = manager.to_prompt()
|
||||
|
||||
# Activate a specific skill
|
||||
skill = manager.get_skill("pdf-processing")
|
||||
manager.activate(skill)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the SkillManager."""
|
||||
self._skills: Dict[str, LoadedSkill] = {}
|
||||
self._loader = SkillLoader()
|
||||
self._discovered = False
|
||||
|
||||
@property
|
||||
def skills(self) -> List[LoadedSkill]:
|
||||
"""Get all loaded skills."""
|
||||
return list(self._skills.values())
|
||||
|
||||
@property
|
||||
def skill_names(self) -> List[str]:
|
||||
"""Get names of all loaded skills."""
|
||||
return list(self._skills.keys())
|
||||
|
||||
def discover(
|
||||
self,
|
||||
skill_dirs: Optional[List[str]] = None,
|
||||
include_defaults: bool = True,
|
||||
) -> int:
|
||||
"""Discover skills from directories.
|
||||
|
||||
Args:
|
||||
skill_dirs: List of directory paths to scan
|
||||
include_defaults: Whether to include default skill directories
|
||||
|
||||
Returns:
|
||||
Number of skills discovered
|
||||
"""
|
||||
props_list = discover_skills(skill_dirs, include_defaults)
|
||||
|
||||
for props in props_list:
|
||||
if props.name not in self._skills:
|
||||
self._skills[props.name] = LoadedSkill(properties=props)
|
||||
|
||||
self._discovered = True
|
||||
return len(props_list)
|
||||
|
||||
def add_skill(self, skill_path: str) -> Optional[LoadedSkill]:
|
||||
"""Add a single skill from a directory path.
|
||||
|
||||
Args:
|
||||
skill_path: Path to skill directory
|
||||
|
||||
Returns:
|
||||
LoadedSkill if successful, None otherwise
|
||||
"""
|
||||
skill = self._loader.load_metadata(skill_path)
|
||||
if skill:
|
||||
self._skills[skill.properties.name] = skill
|
||||
return skill
|
||||
|
||||
def get_skill(self, name: str) -> Optional[LoadedSkill]:
|
||||
"""Get a skill by name.
|
||||
|
||||
Args:
|
||||
name: Skill name
|
||||
|
||||
Returns:
|
||||
LoadedSkill or None if not found
|
||||
"""
|
||||
return self._skills.get(name)
|
||||
|
||||
def activate(self, skill: LoadedSkill) -> bool:
|
||||
"""Activate a skill (load its instructions).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill to activate
|
||||
|
||||
Returns:
|
||||
True if activation succeeded
|
||||
"""
|
||||
return self._loader.activate(skill)
|
||||
|
||||
def activate_by_name(self, name: str) -> bool:
|
||||
"""Activate a skill by name.
|
||||
|
||||
Args:
|
||||
name: Skill name
|
||||
|
||||
Returns:
|
||||
True if activation succeeded
|
||||
"""
|
||||
skill = self.get_skill(name)
|
||||
if skill is None:
|
||||
return False
|
||||
return self.activate(skill)
|
||||
|
||||
def get_available_skills(self) -> List[SkillMetadata]:
|
||||
"""Get metadata for all available skills.
|
||||
|
||||
This is used for system prompt injection.
|
||||
|
||||
Returns:
|
||||
List of SkillMetadata instances
|
||||
"""
|
||||
return [skill.metadata for skill in self._skills.values()]
|
||||
|
||||
def to_prompt(self) -> str:
|
||||
"""Generate XML prompt for available skills.
|
||||
|
||||
Returns:
|
||||
XML string with <available_skills> block
|
||||
"""
|
||||
metadata_list = self.get_available_skills()
|
||||
return generate_skills_xml(metadata_list)
|
||||
|
||||
def get_instructions(self, name: str) -> Optional[str]:
|
||||
"""Get instructions for a skill, activating if needed.
|
||||
|
||||
Args:
|
||||
name: Skill name
|
||||
|
||||
Returns:
|
||||
Skill instructions or None if not found
|
||||
"""
|
||||
skill = self.get_skill(name)
|
||||
if skill is None:
|
||||
return None
|
||||
|
||||
if not skill.is_activated:
|
||||
self.activate(skill)
|
||||
|
||||
return skill.instructions
|
||||
|
||||
def load_resources(self, name: str) -> bool:
|
||||
"""Load all resources for a skill.
|
||||
|
||||
Args:
|
||||
name: Skill name
|
||||
|
||||
Returns:
|
||||
True if resources loaded successfully
|
||||
"""
|
||||
skill = self.get_skill(name)
|
||||
if skill is None:
|
||||
return False
|
||||
|
||||
self._loader.load_all_resources(skill)
|
||||
return True
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear all loaded skills."""
|
||||
self._skills.clear()
|
||||
self._discovered = False
|
||||
|
||||
def __len__(self) -> int:
|
||||
"""Get number of loaded skills."""
|
||||
return len(self._skills)
|
||||
|
||||
def __contains__(self, name: str) -> bool:
|
||||
"""Check if a skill is loaded."""
|
||||
return name in self._skills
|
||||
|
||||
def __iter__(self):
|
||||
"""Iterate over loaded skills."""
|
||||
return iter(self._skills.values())
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Data models for Agent Skills."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class ParseError(Exception):
|
||||
"""Raised when SKILL.md parsing fails."""
|
||||
pass
|
||||
|
||||
|
||||
class ValidationError(Exception):
|
||||
"""Raised when skill validation fails."""
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class SkillProperties:
|
||||
"""Properties parsed from a skill's SKILL.md frontmatter.
|
||||
|
||||
Attributes:
|
||||
name: Skill name in kebab-case (required)
|
||||
description: What the skill does and when the model should use it (required)
|
||||
license: License for the skill (optional)
|
||||
compatibility: Compatibility information for the skill (optional)
|
||||
allowed_tools: Tool patterns the skill requires (optional, experimental)
|
||||
metadata: Key-value pairs for client-specific properties (defaults to
|
||||
empty dict; omitted from to_dict() output when empty)
|
||||
path: Path to the skill directory (optional, added for PraisonAI)
|
||||
"""
|
||||
|
||||
name: str
|
||||
description: str
|
||||
license: Optional[str] = None
|
||||
compatibility: Optional[str] = None
|
||||
allowed_tools: Optional[str] = None
|
||||
metadata: dict = field(default_factory=dict)
|
||||
path: Optional[Path] = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Convert to dictionary, excluding None values and empty metadata."""
|
||||
result = {"name": self.name, "description": self.description}
|
||||
if self.license is not None:
|
||||
result["license"] = self.license
|
||||
if self.compatibility is not None:
|
||||
result["compatibility"] = self.compatibility
|
||||
if self.allowed_tools is not None:
|
||||
result["allowed-tools"] = self.allowed_tools
|
||||
if self.metadata:
|
||||
result["metadata"] = self.metadata
|
||||
return result
|
||||
|
||||
|
||||
@dataclass
|
||||
class SkillMetadata:
|
||||
"""Lightweight skill metadata for system prompt injection.
|
||||
|
||||
This is the minimal information needed to include a skill in the
|
||||
system prompt's <available_skills> block (~50-100 tokens per skill).
|
||||
|
||||
Attributes:
|
||||
name: Skill name
|
||||
description: What the skill does and when to use it
|
||||
location: Absolute path to the SKILL.md file
|
||||
"""
|
||||
|
||||
name: str
|
||||
description: str
|
||||
location: str
|
||||
|
||||
@classmethod
|
||||
def from_properties(cls, props: SkillProperties) -> "SkillMetadata":
|
||||
"""Create SkillMetadata from SkillProperties.
|
||||
|
||||
Args:
|
||||
props: SkillProperties instance with path set
|
||||
|
||||
Returns:
|
||||
SkillMetadata with location pointing to SKILL.md
|
||||
"""
|
||||
if props.path:
|
||||
# Look for SKILL.md or skill.md
|
||||
skill_md = props.path / "SKILL.md"
|
||||
if not skill_md.exists():
|
||||
skill_md = props.path / "skill.md"
|
||||
location = str(skill_md)
|
||||
else:
|
||||
location = ""
|
||||
|
||||
return cls(
|
||||
name=props.name,
|
||||
description=props.description,
|
||||
location=location
|
||||
)
|
||||
@@ -0,0 +1,165 @@
|
||||
"""YAML frontmatter parsing for SKILL.md files."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Optional, Tuple
|
||||
import re
|
||||
|
||||
from .models import ParseError, ValidationError, SkillProperties
|
||||
|
||||
|
||||
def find_skill_md(skill_dir: Path) -> Optional[Path]:
|
||||
"""Find the SKILL.md file in a skill directory.
|
||||
|
||||
Prefers SKILL.md (uppercase) but accepts skill.md (lowercase).
|
||||
|
||||
Args:
|
||||
skill_dir: Path to the skill directory
|
||||
|
||||
Returns:
|
||||
Path to the SKILL.md file, or None if not found
|
||||
"""
|
||||
for name in ("SKILL.md", "skill.md"):
|
||||
path = skill_dir / name
|
||||
if path.exists():
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
def parse_frontmatter(content: str) -> Tuple[dict, str]:
|
||||
"""Parse YAML frontmatter from SKILL.md content.
|
||||
|
||||
Args:
|
||||
content: Raw content of SKILL.md file
|
||||
|
||||
Returns:
|
||||
Tuple of (metadata dict, markdown body)
|
||||
|
||||
Raises:
|
||||
ParseError: If frontmatter is missing or invalid
|
||||
"""
|
||||
if not content.startswith("---"):
|
||||
raise ParseError("SKILL.md must start with YAML frontmatter (---)")
|
||||
|
||||
parts = content.split("---", 2)
|
||||
if len(parts) < 3:
|
||||
raise ParseError("SKILL.md frontmatter not properly closed with ---")
|
||||
|
||||
frontmatter_str = parts[1]
|
||||
body = parts[2].strip()
|
||||
|
||||
try:
|
||||
metadata = _parse_yaml(frontmatter_str)
|
||||
except Exception as e:
|
||||
raise ParseError(f"Invalid YAML in frontmatter: {e}")
|
||||
|
||||
if not isinstance(metadata, dict):
|
||||
raise ParseError("SKILL.md frontmatter must be a YAML mapping")
|
||||
|
||||
# Convert metadata values to strings where needed
|
||||
if "metadata" in metadata and isinstance(metadata["metadata"], dict):
|
||||
metadata["metadata"] = {str(k): str(v) for k, v in metadata["metadata"].items()}
|
||||
|
||||
return metadata, body
|
||||
|
||||
|
||||
def _parse_yaml(yaml_str: str) -> dict:
|
||||
"""Parse YAML string to dictionary.
|
||||
|
||||
Uses a simple parser to avoid strictyaml dependency.
|
||||
Falls back to yaml if available.
|
||||
|
||||
Args:
|
||||
yaml_str: YAML string to parse
|
||||
|
||||
Returns:
|
||||
Parsed dictionary
|
||||
"""
|
||||
# Try to use PyYAML if available
|
||||
try:
|
||||
import yaml
|
||||
return yaml.safe_load(yaml_str) or {}
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Simple fallback parser for basic YAML
|
||||
result = {}
|
||||
current_key = None
|
||||
current_dict = None
|
||||
|
||||
for line in yaml_str.strip().split("\n"):
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
# Check for nested dict
|
||||
if line.startswith(" ") and current_dict is not None:
|
||||
match = re.match(r"^\s+(\w+):\s*(.*)$", line)
|
||||
if match:
|
||||
key, value = match.groups()
|
||||
current_dict[key] = value.strip().strip('"').strip("'")
|
||||
continue
|
||||
|
||||
# Top-level key
|
||||
match = re.match(r"^(\S+):\s*(.*)$", line)
|
||||
if match:
|
||||
key, value = match.groups()
|
||||
value = value.strip()
|
||||
|
||||
if not value:
|
||||
# Could be start of nested dict
|
||||
result[key] = {}
|
||||
current_key = key
|
||||
current_dict = result[key]
|
||||
else:
|
||||
result[key] = value.strip('"').strip("'")
|
||||
current_dict = None
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def read_properties(skill_dir: Path) -> SkillProperties:
|
||||
"""Read skill properties from SKILL.md frontmatter.
|
||||
|
||||
This function parses the frontmatter and returns properties.
|
||||
It does NOT perform full validation. Use validate() for that.
|
||||
|
||||
Args:
|
||||
skill_dir: Path to the skill directory
|
||||
|
||||
Returns:
|
||||
SkillProperties with parsed metadata
|
||||
|
||||
Raises:
|
||||
ParseError: If SKILL.md is missing or has invalid YAML
|
||||
ValidationError: If required fields (name, description) are missing
|
||||
"""
|
||||
skill_dir = Path(skill_dir)
|
||||
skill_md = find_skill_md(skill_dir)
|
||||
|
||||
if skill_md is None:
|
||||
raise ParseError(f"SKILL.md not found in {skill_dir}")
|
||||
|
||||
content = skill_md.read_text()
|
||||
metadata, _ = parse_frontmatter(content)
|
||||
|
||||
if "name" not in metadata:
|
||||
raise ValidationError("Missing required field in frontmatter: name")
|
||||
if "description" not in metadata:
|
||||
raise ValidationError("Missing required field in frontmatter: description")
|
||||
|
||||
name = metadata["name"]
|
||||
description = metadata["description"]
|
||||
|
||||
if not isinstance(name, str) or not name.strip():
|
||||
raise ValidationError("Field 'name' must be a non-empty string")
|
||||
if not isinstance(description, str) or not description.strip():
|
||||
raise ValidationError("Field 'description' must be a non-empty string")
|
||||
|
||||
return SkillProperties(
|
||||
name=name.strip(),
|
||||
description=description.strip(),
|
||||
license=metadata.get("license"),
|
||||
compatibility=metadata.get("compatibility"),
|
||||
allowed_tools=metadata.get("allowed-tools"),
|
||||
metadata=metadata.get("metadata") or {},
|
||||
path=skill_dir,
|
||||
)
|
||||
@@ -0,0 +1,74 @@
|
||||
"""XML prompt generation for Agent Skills."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
import html
|
||||
|
||||
from .models import SkillMetadata
|
||||
from .parser import read_properties
|
||||
|
||||
|
||||
def format_skill_for_prompt(skill: SkillMetadata) -> str:
|
||||
"""Format a single skill as XML for system prompt.
|
||||
|
||||
Args:
|
||||
skill: SkillMetadata instance
|
||||
|
||||
Returns:
|
||||
XML string for the skill
|
||||
"""
|
||||
name = html.escape(skill.name)
|
||||
description = html.escape(skill.description)
|
||||
location = html.escape(skill.location)
|
||||
|
||||
return f""" <skill>
|
||||
<name>{name}</name>
|
||||
<description>{description}</description>
|
||||
<location>{location}</location>
|
||||
</skill>"""
|
||||
|
||||
|
||||
def generate_skills_xml(skills: List[SkillMetadata]) -> str:
|
||||
"""Generate XML block for available skills.
|
||||
|
||||
Args:
|
||||
skills: List of SkillMetadata instances
|
||||
|
||||
Returns:
|
||||
XML string with <available_skills> block
|
||||
"""
|
||||
if not skills:
|
||||
return "<available_skills>\n</available_skills>"
|
||||
|
||||
skill_entries = "\n".join(format_skill_for_prompt(s) for s in skills)
|
||||
return f"<available_skills>\n{skill_entries}\n</available_skills>"
|
||||
|
||||
|
||||
def to_prompt(skill_dirs: List[Path]) -> str:
|
||||
"""Generate prompt XML from skill directories.
|
||||
|
||||
This is the main entry point for generating the <available_skills>
|
||||
block to inject into agent system prompts.
|
||||
|
||||
Args:
|
||||
skill_dirs: List of paths to skill directories
|
||||
|
||||
Returns:
|
||||
XML string with <available_skills> block
|
||||
"""
|
||||
skills = []
|
||||
|
||||
for skill_dir in skill_dirs:
|
||||
skill_dir = Path(skill_dir)
|
||||
if not skill_dir.exists() or not skill_dir.is_dir():
|
||||
continue
|
||||
|
||||
try:
|
||||
props = read_properties(skill_dir)
|
||||
meta = SkillMetadata.from_properties(props)
|
||||
skills.append(meta)
|
||||
except Exception:
|
||||
# Skip invalid skills silently
|
||||
continue
|
||||
|
||||
return generate_skills_xml(skills)
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Skill validation logic."""
|
||||
|
||||
import unicodedata
|
||||
from pathlib import Path
|
||||
from typing import Optional, List
|
||||
|
||||
from .models import ParseError
|
||||
from .parser import find_skill_md, parse_frontmatter
|
||||
|
||||
MAX_SKILL_NAME_LENGTH = 64
|
||||
MAX_DESCRIPTION_LENGTH = 1024
|
||||
MAX_COMPATIBILITY_LENGTH = 500
|
||||
|
||||
ALLOWED_FIELDS = {
|
||||
"name",
|
||||
"description",
|
||||
"license",
|
||||
"allowed-tools",
|
||||
"metadata",
|
||||
"compatibility",
|
||||
}
|
||||
|
||||
|
||||
def _validate_name(name: str, skill_dir: Optional[Path]) -> List[str]:
|
||||
"""Validate skill name format and directory match.
|
||||
|
||||
Skill names support i18n characters (Unicode letters) plus hyphens.
|
||||
Names must be lowercase and cannot start/end with hyphens.
|
||||
"""
|
||||
errors = []
|
||||
|
||||
if not name or not isinstance(name, str) or not name.strip():
|
||||
errors.append("Field 'name' must be a non-empty string")
|
||||
return errors
|
||||
|
||||
name = unicodedata.normalize("NFKC", name.strip())
|
||||
|
||||
if len(name) > MAX_SKILL_NAME_LENGTH:
|
||||
errors.append(
|
||||
f"Skill name '{name}' exceeds {MAX_SKILL_NAME_LENGTH} character limit "
|
||||
f"({len(name)} chars)"
|
||||
)
|
||||
|
||||
if name != name.lower():
|
||||
errors.append(f"Skill name '{name}' must be lowercase")
|
||||
|
||||
if name.startswith("-") or name.endswith("-"):
|
||||
errors.append("Skill name cannot start or end with a hyphen")
|
||||
|
||||
if "--" in name:
|
||||
errors.append("Skill name cannot contain consecutive hyphens")
|
||||
|
||||
if not all(c.isalnum() or c == "-" for c in name):
|
||||
errors.append(
|
||||
f"Skill name '{name}' contains invalid characters. "
|
||||
"Only letters, digits, and hyphens are allowed."
|
||||
)
|
||||
|
||||
if skill_dir:
|
||||
dir_name = unicodedata.normalize("NFKC", skill_dir.name)
|
||||
if dir_name != name:
|
||||
errors.append(
|
||||
f"Directory name '{skill_dir.name}' must match skill name '{name}'"
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def _validate_description(description: str) -> List[str]:
|
||||
"""Validate description format."""
|
||||
errors = []
|
||||
|
||||
if not description or not isinstance(description, str) or not description.strip():
|
||||
errors.append("Field 'description' must be a non-empty string")
|
||||
return errors
|
||||
|
||||
if len(description) > MAX_DESCRIPTION_LENGTH:
|
||||
errors.append(
|
||||
f"Description exceeds {MAX_DESCRIPTION_LENGTH} character limit "
|
||||
f"({len(description)} chars)"
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def _validate_compatibility(compatibility: str) -> List[str]:
|
||||
"""Validate compatibility format."""
|
||||
errors = []
|
||||
|
||||
if not isinstance(compatibility, str):
|
||||
errors.append("Field 'compatibility' must be a string")
|
||||
return errors
|
||||
|
||||
if len(compatibility) > MAX_COMPATIBILITY_LENGTH:
|
||||
errors.append(
|
||||
f"Compatibility exceeds {MAX_COMPATIBILITY_LENGTH} character limit "
|
||||
f"({len(compatibility)} chars)"
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def _validate_metadata_fields(metadata: dict) -> List[str]:
|
||||
"""Validate that only allowed fields are present."""
|
||||
errors = []
|
||||
|
||||
extra_fields = set(metadata.keys()) - ALLOWED_FIELDS
|
||||
if extra_fields:
|
||||
errors.append(
|
||||
f"Unexpected fields in frontmatter: {', '.join(sorted(extra_fields))}. "
|
||||
f"Only {sorted(ALLOWED_FIELDS)} are allowed."
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def validate_metadata(metadata: dict, skill_dir: Optional[Path] = None) -> List[str]:
|
||||
"""Validate parsed skill metadata.
|
||||
|
||||
This is the core validation function that works on already-parsed metadata,
|
||||
avoiding duplicate file I/O when called from the parser.
|
||||
|
||||
Args:
|
||||
metadata: Parsed YAML frontmatter dictionary
|
||||
skill_dir: Optional path to skill directory (for name-directory match check)
|
||||
|
||||
Returns:
|
||||
List of validation error messages. Empty list means valid.
|
||||
"""
|
||||
errors = []
|
||||
errors.extend(_validate_metadata_fields(metadata))
|
||||
|
||||
if "name" not in metadata:
|
||||
errors.append("Missing required field in frontmatter: name")
|
||||
else:
|
||||
errors.extend(_validate_name(metadata["name"], skill_dir))
|
||||
|
||||
if "description" not in metadata:
|
||||
errors.append("Missing required field in frontmatter: description")
|
||||
else:
|
||||
errors.extend(_validate_description(metadata["description"]))
|
||||
|
||||
if "compatibility" in metadata:
|
||||
errors.extend(_validate_compatibility(metadata["compatibility"]))
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def validate(skill_dir: Path) -> List[str]:
|
||||
"""Validate a skill directory.
|
||||
|
||||
Args:
|
||||
skill_dir: Path to the skill directory
|
||||
|
||||
Returns:
|
||||
List of validation error messages. Empty list means valid.
|
||||
"""
|
||||
skill_dir = Path(skill_dir)
|
||||
|
||||
if not skill_dir.exists():
|
||||
return [f"Path does not exist: {skill_dir}"]
|
||||
|
||||
if not skill_dir.is_dir():
|
||||
return [f"Not a directory: {skill_dir}"]
|
||||
|
||||
skill_md = find_skill_md(skill_dir)
|
||||
if skill_md is None:
|
||||
return ["Missing required file: SKILL.md"]
|
||||
|
||||
try:
|
||||
content = skill_md.read_text()
|
||||
metadata, _ = parse_frontmatter(content)
|
||||
except ParseError as e:
|
||||
return [str(e)]
|
||||
|
||||
return validate_metadata(metadata, skill_dir)
|
||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "praisonaiagents"
|
||||
version = "0.1.21"
|
||||
version = "0.1.23"
|
||||
description = "Praison AI agents for completing complex tasks with Self Reflection Agents"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Unit tests for PraisonAI Agents evaluation framework."""
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Unit tests for BaseEvaluator class."""
|
||||
|
||||
import pytest
|
||||
import tempfile
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from praisonaiagents.eval.base import BaseEvaluator
|
||||
|
||||
|
||||
class ConcreteEvaluator(BaseEvaluator):
|
||||
"""Concrete implementation for testing."""
|
||||
|
||||
def run(self, **kwargs):
|
||||
return {"test": "result"}
|
||||
|
||||
|
||||
class TestBaseEvaluator:
|
||||
"""Tests for BaseEvaluator abstract class."""
|
||||
|
||||
def test_init_default_values(self):
|
||||
"""Test default initialization values."""
|
||||
evaluator = ConcreteEvaluator()
|
||||
assert evaluator.name.startswith("eval_")
|
||||
assert evaluator.eval_id is not None
|
||||
assert len(evaluator.eval_id) == 8
|
||||
assert evaluator.save_results_path is None
|
||||
assert evaluator.verbose is False
|
||||
|
||||
def test_init_custom_values(self):
|
||||
"""Test initialization with custom values."""
|
||||
evaluator = ConcreteEvaluator(
|
||||
name="my_eval",
|
||||
save_results_path="/tmp/results.json",
|
||||
verbose=True
|
||||
)
|
||||
assert evaluator.name == "my_eval"
|
||||
assert evaluator.save_results_path == "/tmp/results.json"
|
||||
assert evaluator.verbose is True
|
||||
|
||||
def test_eval_id_unique(self):
|
||||
"""Test that each evaluator gets a unique ID."""
|
||||
eval1 = ConcreteEvaluator()
|
||||
eval2 = ConcreteEvaluator()
|
||||
assert eval1.eval_id != eval2.eval_id
|
||||
|
||||
def test_before_run_verbose(self):
|
||||
"""Test before_run hook with verbose logging."""
|
||||
evaluator = ConcreteEvaluator(verbose=True)
|
||||
with patch('praisonaiagents.eval.base.logger') as mock_logger:
|
||||
evaluator.before_run()
|
||||
mock_logger.info.assert_called()
|
||||
|
||||
def test_after_run_verbose(self):
|
||||
"""Test after_run hook with verbose logging."""
|
||||
evaluator = ConcreteEvaluator(verbose=True)
|
||||
with patch('praisonaiagents.eval.base.logger') as mock_logger:
|
||||
evaluator.after_run({"result": "test"})
|
||||
mock_logger.info.assert_called()
|
||||
|
||||
def test_save_result_to_file(self):
|
||||
"""Test saving results to file."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/results.json"
|
||||
evaluator = ConcreteEvaluator(
|
||||
name="test_save",
|
||||
save_results_path=path
|
||||
)
|
||||
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
evaluator._save_result(result)
|
||||
|
||||
assert Path(path).exists()
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
assert data["score"] == 8.0
|
||||
|
||||
def test_save_result_with_placeholders(self):
|
||||
"""Test saving results with path placeholders."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/{{name}}_{{eval_id}}.json"
|
||||
evaluator = ConcreteEvaluator(
|
||||
name="test_eval",
|
||||
save_results_path=path
|
||||
)
|
||||
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
evaluator._save_result(result)
|
||||
|
||||
expected_path = f"{tmpdir}/test_eval_{evaluator.eval_id}.json"
|
||||
assert Path(expected_path).exists()
|
||||
|
||||
def test_save_result_creates_directories(self):
|
||||
"""Test that save creates parent directories."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/nested/dir/results.json"
|
||||
evaluator = ConcreteEvaluator(save_results_path=path)
|
||||
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
evaluator._save_result(result)
|
||||
|
||||
assert Path(path).exists()
|
||||
|
||||
def test_run_abstract_method(self):
|
||||
"""Test that run method works in concrete implementation."""
|
||||
evaluator = ConcreteEvaluator()
|
||||
result = evaluator.run()
|
||||
assert result == {"test": "result"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_async_default(self):
|
||||
"""Test default async run calls sync run."""
|
||||
evaluator = ConcreteEvaluator()
|
||||
result = await evaluator.run_async()
|
||||
assert result == {"test": "result"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_before_run(self):
|
||||
"""Test async before_run hook."""
|
||||
evaluator = ConcreteEvaluator(verbose=True)
|
||||
with patch('praisonaiagents.eval.base.logger') as mock_logger:
|
||||
await evaluator.async_before_run()
|
||||
mock_logger.info.assert_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_after_run(self):
|
||||
"""Test async after_run hook."""
|
||||
evaluator = ConcreteEvaluator(verbose=True)
|
||||
with patch('praisonaiagents.eval.base.logger') as mock_logger:
|
||||
await evaluator.async_after_run({"result": "test"})
|
||||
mock_logger.info.assert_called()
|
||||
|
||||
def test_repr(self):
|
||||
"""Test string representation."""
|
||||
evaluator = ConcreteEvaluator(name="test_eval")
|
||||
repr_str = repr(evaluator)
|
||||
assert "ConcreteEvaluator" in repr_str
|
||||
assert "test_eval" in repr_str
|
||||
assert evaluator.eval_id in repr_str
|
||||
@@ -0,0 +1,425 @@
|
||||
"""Unit tests for evaluator classes."""
|
||||
|
||||
import pytest
|
||||
import time
|
||||
from unittest.mock import MagicMock, patch, AsyncMock
|
||||
|
||||
from praisonaiagents.eval.accuracy import AccuracyEvaluator
|
||||
from praisonaiagents.eval.performance import PerformanceEvaluator
|
||||
from praisonaiagents.eval.reliability import ReliabilityEvaluator
|
||||
from praisonaiagents.eval.criteria import CriteriaEvaluator
|
||||
from praisonaiagents.eval.results import (
|
||||
AccuracyResult,
|
||||
PerformanceResult,
|
||||
ReliabilityResult,
|
||||
CriteriaResult,
|
||||
)
|
||||
|
||||
|
||||
class TestAccuracyEvaluator:
|
||||
"""Tests for AccuracyEvaluator class."""
|
||||
|
||||
def test_init_with_func(self):
|
||||
"""Test initialization with function."""
|
||||
func = lambda x: "test output"
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=func,
|
||||
input_text="test input",
|
||||
expected_output="test output"
|
||||
)
|
||||
assert evaluator.func == func
|
||||
assert evaluator.input_text == "test input"
|
||||
assert evaluator.expected_output == "test output"
|
||||
|
||||
def test_init_with_agent(self):
|
||||
"""Test initialization with agent."""
|
||||
agent = MagicMock()
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
assert evaluator.agent == agent
|
||||
|
||||
def test_init_requires_agent_or_func(self):
|
||||
"""Test that either agent or func must be provided."""
|
||||
with pytest.raises(ValueError, match="Either 'agent' or 'func' must be provided"):
|
||||
AccuracyEvaluator(
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
|
||||
def test_get_output_from_func(self):
|
||||
"""Test getting output from function."""
|
||||
func = lambda x: f"output for {x}"
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=func,
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
output = evaluator._get_output()
|
||||
assert output == "output for test"
|
||||
|
||||
def test_get_output_from_agent_chat(self):
|
||||
"""Test getting output from agent with chat method."""
|
||||
agent = MagicMock()
|
||||
agent.chat.return_value = "agent response"
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
output = evaluator._get_output()
|
||||
assert output == "agent response"
|
||||
agent.chat.assert_called_once_with("test")
|
||||
|
||||
def test_get_output_from_agent_start(self):
|
||||
"""Test getting output from agent with start method."""
|
||||
agent = MagicMock(spec=['start'])
|
||||
result = MagicMock()
|
||||
result.raw = "agent result"
|
||||
agent.start.return_value = result
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
output = evaluator._get_output()
|
||||
assert output == "agent result"
|
||||
|
||||
def test_judge_output(self):
|
||||
"""Test LLM judging of output."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 8\nREASONING: Good match"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "4"
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=test_func,
|
||||
input_text="What is 2+2?",
|
||||
expected_output="4"
|
||||
)
|
||||
|
||||
score = evaluator._judge_output("4")
|
||||
assert score.score == 8.0
|
||||
assert "Good match" in score.reasoning
|
||||
|
||||
def test_run_returns_accuracy_result(self):
|
||||
"""Test that run returns AccuracyResult."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 9\nREASONING: Excellent"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "4"
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=test_func,
|
||||
input_text="What is 2+2?",
|
||||
expected_output="4",
|
||||
num_iterations=2
|
||||
)
|
||||
|
||||
result = evaluator.run()
|
||||
assert isinstance(result, AccuracyResult)
|
||||
assert len(result.evaluations) == 2
|
||||
|
||||
def test_evaluate_output(self):
|
||||
"""Test evaluate_output method for pre-generated outputs."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 10\nREASONING: Perfect"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "unused"
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=test_func,
|
||||
input_text="What is 2+2?",
|
||||
expected_output="4"
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_output("4")
|
||||
assert isinstance(result, AccuracyResult)
|
||||
assert len(result.evaluations) == 1
|
||||
assert result.evaluations[0].score == 10.0
|
||||
|
||||
|
||||
class TestPerformanceEvaluator:
|
||||
"""Tests for PerformanceEvaluator class."""
|
||||
|
||||
def test_init_with_func(self):
|
||||
"""Test initialization with function."""
|
||||
func = lambda: time.sleep(0.01)
|
||||
evaluator = PerformanceEvaluator(
|
||||
func=func,
|
||||
num_iterations=5,
|
||||
warmup_runs=2
|
||||
)
|
||||
assert evaluator.func == func
|
||||
assert evaluator.num_iterations == 5
|
||||
assert evaluator.warmup_runs == 2
|
||||
|
||||
def test_init_with_agent(self):
|
||||
"""Test initialization with agent."""
|
||||
agent = MagicMock()
|
||||
evaluator = PerformanceEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
num_iterations=3
|
||||
)
|
||||
assert evaluator.agent == agent
|
||||
assert evaluator.input_text == "test"
|
||||
|
||||
def test_init_requires_func_or_agent(self):
|
||||
"""Test that either func or agent must be provided."""
|
||||
with pytest.raises(ValueError, match="Either 'func' or 'agent' must be provided"):
|
||||
PerformanceEvaluator()
|
||||
|
||||
def test_run_measures_time(self):
|
||||
"""Test that run measures execution time."""
|
||||
call_count = 0
|
||||
def slow_func():
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
time.sleep(0.01)
|
||||
|
||||
evaluator = PerformanceEvaluator(
|
||||
func=slow_func,
|
||||
num_iterations=3,
|
||||
warmup_runs=1,
|
||||
track_memory=False
|
||||
)
|
||||
|
||||
result = evaluator.run()
|
||||
assert isinstance(result, PerformanceResult)
|
||||
assert len(result.metrics) == 3
|
||||
assert all(m.run_time >= 0.01 for m in result.metrics)
|
||||
assert call_count == 4 # 1 warmup + 3 iterations
|
||||
|
||||
def test_run_with_agent(self):
|
||||
"""Test run with agent."""
|
||||
agent = MagicMock()
|
||||
agent.chat.return_value = "response"
|
||||
|
||||
evaluator = PerformanceEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
num_iterations=2,
|
||||
warmup_runs=1,
|
||||
track_memory=False
|
||||
)
|
||||
|
||||
result = evaluator.run()
|
||||
assert isinstance(result, PerformanceResult)
|
||||
assert len(result.metrics) == 2
|
||||
assert agent.chat.call_count == 3 # 1 warmup + 2 iterations
|
||||
|
||||
|
||||
class TestReliabilityEvaluator:
|
||||
"""Tests for ReliabilityEvaluator class."""
|
||||
|
||||
def test_init(self):
|
||||
"""Test initialization."""
|
||||
agent = MagicMock()
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_tools=["search", "calculate"]
|
||||
)
|
||||
assert evaluator.agent == agent
|
||||
assert evaluator.expected_tools == {"search", "calculate"}
|
||||
|
||||
def test_init_requires_agent(self):
|
||||
"""Test that agent must be provided."""
|
||||
with pytest.raises(ValueError, match="'agent' must be provided"):
|
||||
ReliabilityEvaluator(
|
||||
input_text="test",
|
||||
expected_tools=["search"]
|
||||
)
|
||||
|
||||
def test_evaluate_tool_calls(self):
|
||||
"""Test evaluate_tool_calls method."""
|
||||
agent = MagicMock()
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_tools=["search", "calculate"],
|
||||
forbidden_tools=["delete"]
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_tool_calls(["search", "calculate"])
|
||||
assert isinstance(result, ReliabilityResult)
|
||||
assert result.passed is True
|
||||
assert len(result.passed_calls) == 3 # 2 expected + 1 forbidden not called
|
||||
|
||||
def test_evaluate_tool_calls_with_failures(self):
|
||||
"""Test evaluate_tool_calls with missing tools."""
|
||||
agent = MagicMock()
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_tools=["search", "calculate"]
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_tool_calls(["search"]) # missing calculate
|
||||
assert result.passed is False
|
||||
assert len(result.failed_calls) == 1
|
||||
|
||||
def test_forbidden_tools(self):
|
||||
"""Test forbidden tools detection."""
|
||||
agent = MagicMock()
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_tools=[],
|
||||
forbidden_tools=["delete", "drop"]
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_tool_calls(["delete"]) # called forbidden tool
|
||||
assert result.passed is False
|
||||
assert len(result.failed_calls) == 1
|
||||
|
||||
|
||||
class TestCriteriaEvaluator:
|
||||
"""Tests for CriteriaEvaluator class."""
|
||||
|
||||
def test_init(self):
|
||||
"""Test initialization."""
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Response is helpful",
|
||||
func=lambda x: "test",
|
||||
input_text="test"
|
||||
)
|
||||
assert evaluator.criteria == "Response is helpful"
|
||||
assert evaluator.scoring_type == "numeric"
|
||||
assert evaluator.threshold == 7.0
|
||||
|
||||
def test_init_binary_scoring(self):
|
||||
"""Test initialization with binary scoring."""
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Response is helpful",
|
||||
func=lambda x: "test",
|
||||
scoring_type="binary"
|
||||
)
|
||||
assert evaluator.scoring_type == "binary"
|
||||
|
||||
def test_judge_output_numeric(self):
|
||||
"""Test numeric scoring."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 8\nREASONING: Good"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "test"
|
||||
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Be helpful",
|
||||
func=test_func,
|
||||
scoring_type="numeric",
|
||||
threshold=7.0
|
||||
)
|
||||
|
||||
score = evaluator._judge_output("test output")
|
||||
assert score.score == 8.0
|
||||
assert score.passed is True
|
||||
|
||||
def test_judge_output_binary(self):
|
||||
"""Test binary scoring."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "RESULT: PASS\nREASONING: Meets criteria"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "test"
|
||||
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Be helpful",
|
||||
func=test_func,
|
||||
scoring_type="binary"
|
||||
)
|
||||
|
||||
score = evaluator._judge_output("test output")
|
||||
assert score.score == 10.0
|
||||
assert score.passed is True
|
||||
|
||||
def test_on_fail_callback(self):
|
||||
"""Test on_fail callback is called."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 3\nREASONING: Poor"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
on_fail_called = []
|
||||
def on_fail(score):
|
||||
on_fail_called.append(score)
|
||||
|
||||
def test_func(x):
|
||||
return "test"
|
||||
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Be helpful",
|
||||
func=test_func,
|
||||
threshold=7.0,
|
||||
on_fail=on_fail
|
||||
)
|
||||
|
||||
evaluator.run()
|
||||
assert len(on_fail_called) == 1
|
||||
assert on_fail_called[0].score == 3.0
|
||||
|
||||
def test_evaluate_output_criteria(self):
|
||||
"""Test evaluate_output method."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 9\nREASONING: Great"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "unused"
|
||||
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Be helpful",
|
||||
func=test_func
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_output("pre-generated output")
|
||||
assert isinstance(result, CriteriaResult)
|
||||
assert len(result.evaluations) == 1
|
||||
@@ -0,0 +1,359 @@
|
||||
"""Unit tests for evaluation result dataclasses."""
|
||||
|
||||
import pytest
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
from praisonaiagents.eval.results import (
|
||||
EvaluationScore,
|
||||
AccuracyResult,
|
||||
PerformanceMetrics,
|
||||
PerformanceResult,
|
||||
ToolCallResult,
|
||||
ReliabilityResult,
|
||||
CriteriaScore,
|
||||
CriteriaResult,
|
||||
)
|
||||
|
||||
|
||||
class TestEvaluationScore:
|
||||
"""Tests for EvaluationScore dataclass."""
|
||||
|
||||
def test_creation(self):
|
||||
"""Test basic creation of EvaluationScore."""
|
||||
score = EvaluationScore(
|
||||
score=8.5,
|
||||
reasoning="Good response",
|
||||
input_text="What is 2+2?",
|
||||
output_text="4",
|
||||
expected_output="4"
|
||||
)
|
||||
assert score.score == 8.5
|
||||
assert score.reasoning == "Good response"
|
||||
assert score.input_text == "What is 2+2?"
|
||||
assert score.output_text == "4"
|
||||
assert score.expected_output == "4"
|
||||
|
||||
def test_timestamp_auto_generated(self):
|
||||
"""Test that timestamp is auto-generated."""
|
||||
score = EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Test",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
)
|
||||
assert score.timestamp is not None
|
||||
datetime.fromisoformat(score.timestamp)
|
||||
|
||||
def test_metadata_default(self):
|
||||
"""Test that metadata defaults to empty dict."""
|
||||
score = EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Test",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
)
|
||||
assert score.metadata == {}
|
||||
|
||||
|
||||
class TestAccuracyResult:
|
||||
"""Tests for AccuracyResult dataclass."""
|
||||
|
||||
def test_empty_result(self):
|
||||
"""Test empty AccuracyResult."""
|
||||
result = AccuracyResult()
|
||||
assert result.evaluations == []
|
||||
assert result.avg_score == 0.0
|
||||
assert result.min_score == 0.0
|
||||
assert result.max_score == 0.0
|
||||
assert result.std_dev == 0.0
|
||||
assert result.passed is False
|
||||
|
||||
def test_single_evaluation(self):
|
||||
"""Test AccuracyResult with single evaluation."""
|
||||
result = AccuracyResult()
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Good",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
assert result.avg_score == 8.0
|
||||
assert result.min_score == 8.0
|
||||
assert result.max_score == 8.0
|
||||
assert result.std_dev == 0.0
|
||||
assert result.passed is True
|
||||
|
||||
def test_multiple_evaluations(self):
|
||||
"""Test AccuracyResult with multiple evaluations."""
|
||||
result = AccuracyResult()
|
||||
for score in [6.0, 8.0, 10.0]:
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=score,
|
||||
reasoning="Test",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
assert result.avg_score == 8.0
|
||||
assert result.min_score == 6.0
|
||||
assert result.max_score == 10.0
|
||||
assert result.passed is True
|
||||
|
||||
def test_passed_threshold(self):
|
||||
"""Test passed property with different scores."""
|
||||
result = AccuracyResult()
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=6.0,
|
||||
reasoning="Below threshold",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
assert result.passed is False
|
||||
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Above threshold",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
assert result.passed is True
|
||||
|
||||
def test_to_dict(self):
|
||||
"""Test to_dict method."""
|
||||
result = AccuracyResult(eval_id="test123", name="test_eval")
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Good",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
d = result.to_dict()
|
||||
assert d["eval_id"] == "test123"
|
||||
assert d["name"] == "test_eval"
|
||||
assert d["avg_score"] == 8.0
|
||||
assert d["num_evaluations"] == 1
|
||||
|
||||
def test_to_json(self):
|
||||
"""Test to_json method."""
|
||||
result = AccuracyResult(eval_id="test123")
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Good",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
json_str = result.to_json()
|
||||
data = json.loads(json_str)
|
||||
assert data["eval_id"] == "test123"
|
||||
|
||||
|
||||
class TestPerformanceMetrics:
|
||||
"""Tests for PerformanceMetrics dataclass."""
|
||||
|
||||
def test_creation(self):
|
||||
"""Test basic creation."""
|
||||
metrics = PerformanceMetrics(
|
||||
run_time=0.5,
|
||||
memory_usage=10.5,
|
||||
iteration=1
|
||||
)
|
||||
assert metrics.run_time == 0.5
|
||||
assert metrics.memory_usage == 10.5
|
||||
assert metrics.iteration == 1
|
||||
|
||||
|
||||
class TestPerformanceResult:
|
||||
"""Tests for PerformanceResult dataclass."""
|
||||
|
||||
def test_empty_result(self):
|
||||
"""Test empty PerformanceResult."""
|
||||
result = PerformanceResult()
|
||||
assert result.metrics == []
|
||||
assert result.avg_run_time == 0.0
|
||||
assert result.min_run_time == 0.0
|
||||
assert result.max_run_time == 0.0
|
||||
assert result.avg_memory == 0.0
|
||||
|
||||
def test_with_metrics(self):
|
||||
"""Test PerformanceResult with metrics."""
|
||||
result = PerformanceResult()
|
||||
for i, (time, mem) in enumerate([(0.1, 5.0), (0.2, 10.0), (0.3, 15.0)]):
|
||||
result.metrics.append(PerformanceMetrics(
|
||||
run_time=time,
|
||||
memory_usage=mem,
|
||||
iteration=i + 1
|
||||
))
|
||||
assert result.avg_run_time == 0.2
|
||||
assert result.min_run_time == 0.1
|
||||
assert result.max_run_time == 0.3
|
||||
assert result.avg_memory == 10.0
|
||||
assert result.max_memory == 15.0
|
||||
|
||||
def test_median_and_p95(self):
|
||||
"""Test median and p95 calculations."""
|
||||
result = PerformanceResult()
|
||||
for i in range(100):
|
||||
result.metrics.append(PerformanceMetrics(
|
||||
run_time=float(i + 1) / 100,
|
||||
memory_usage=1.0,
|
||||
iteration=i + 1
|
||||
))
|
||||
assert 0.49 <= result.median_run_time <= 0.51
|
||||
assert result.p95_run_time >= 0.95
|
||||
|
||||
def test_to_dict(self):
|
||||
"""Test to_dict method."""
|
||||
result = PerformanceResult(warmup_runs=5, eval_id="perf123")
|
||||
result.metrics.append(PerformanceMetrics(
|
||||
run_time=0.5,
|
||||
memory_usage=10.0,
|
||||
iteration=1
|
||||
))
|
||||
d = result.to_dict()
|
||||
assert d["warmup_runs"] == 5
|
||||
assert d["eval_id"] == "perf123"
|
||||
assert d["num_iterations"] == 1
|
||||
|
||||
|
||||
class TestToolCallResult:
|
||||
"""Tests for ToolCallResult dataclass."""
|
||||
|
||||
def test_creation(self):
|
||||
"""Test basic creation."""
|
||||
tc = ToolCallResult(
|
||||
tool_name="search",
|
||||
expected=True,
|
||||
actual=True,
|
||||
passed=True
|
||||
)
|
||||
assert tc.tool_name == "search"
|
||||
assert tc.expected is True
|
||||
assert tc.actual is True
|
||||
assert tc.passed is True
|
||||
|
||||
|
||||
class TestReliabilityResult:
|
||||
"""Tests for ReliabilityResult dataclass."""
|
||||
|
||||
def test_empty_result(self):
|
||||
"""Test empty ReliabilityResult."""
|
||||
result = ReliabilityResult()
|
||||
assert result.tool_results == []
|
||||
assert result.pass_rate == 0.0
|
||||
assert result.status == "PASSED"
|
||||
assert result.passed is True
|
||||
|
||||
def test_with_tool_calls(self):
|
||||
"""Test ReliabilityResult with tool calls."""
|
||||
result = ReliabilityResult()
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name="search",
|
||||
expected=True,
|
||||
actual=True,
|
||||
passed=True
|
||||
))
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name="calculate",
|
||||
expected=True,
|
||||
actual=False,
|
||||
passed=False
|
||||
))
|
||||
assert len(result.passed_calls) == 1
|
||||
assert len(result.failed_calls) == 1
|
||||
assert result.pass_rate == 0.5
|
||||
assert result.status == "FAILED"
|
||||
assert result.passed is False
|
||||
|
||||
def test_all_passed(self):
|
||||
"""Test when all tool calls pass."""
|
||||
result = ReliabilityResult()
|
||||
for name in ["tool1", "tool2", "tool3"]:
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=name,
|
||||
expected=True,
|
||||
actual=True,
|
||||
passed=True
|
||||
))
|
||||
assert result.pass_rate == 1.0
|
||||
assert result.passed is True
|
||||
|
||||
|
||||
class TestCriteriaScore:
|
||||
"""Tests for CriteriaScore dataclass."""
|
||||
|
||||
def test_creation(self):
|
||||
"""Test basic creation."""
|
||||
score = CriteriaScore(
|
||||
score=8.0,
|
||||
passed=True,
|
||||
reasoning="Meets criteria",
|
||||
output_text="test output",
|
||||
criteria="Be helpful"
|
||||
)
|
||||
assert score.score == 8.0
|
||||
assert score.passed is True
|
||||
assert score.reasoning == "Meets criteria"
|
||||
|
||||
|
||||
class TestCriteriaResult:
|
||||
"""Tests for CriteriaResult dataclass."""
|
||||
|
||||
def test_empty_result(self):
|
||||
"""Test empty CriteriaResult."""
|
||||
result = CriteriaResult(criteria="Be helpful")
|
||||
assert result.evaluations == []
|
||||
assert result.avg_score == 0.0
|
||||
assert result.pass_rate == 0.0
|
||||
|
||||
def test_numeric_scoring(self):
|
||||
"""Test numeric scoring mode."""
|
||||
result = CriteriaResult(
|
||||
criteria="Be helpful",
|
||||
scoring_type="numeric",
|
||||
threshold=7.0
|
||||
)
|
||||
result.evaluations.append(CriteriaScore(
|
||||
score=8.0,
|
||||
passed=True,
|
||||
reasoning="Good",
|
||||
output_text="test",
|
||||
criteria="Be helpful"
|
||||
))
|
||||
assert result.avg_score == 8.0
|
||||
assert result.passed is True
|
||||
|
||||
def test_binary_scoring(self):
|
||||
"""Test binary scoring mode."""
|
||||
result = CriteriaResult(
|
||||
criteria="Be helpful",
|
||||
scoring_type="binary"
|
||||
)
|
||||
result.evaluations.append(CriteriaScore(
|
||||
score=10.0,
|
||||
passed=True,
|
||||
reasoning="Pass",
|
||||
output_text="test",
|
||||
criteria="Be helpful"
|
||||
))
|
||||
result.evaluations.append(CriteriaScore(
|
||||
score=0.0,
|
||||
passed=False,
|
||||
reasoning="Fail",
|
||||
output_text="test",
|
||||
criteria="Be helpful"
|
||||
))
|
||||
assert result.pass_rate == 0.5
|
||||
assert result.passed is True
|
||||
|
||||
def test_to_dict(self):
|
||||
"""Test to_dict method."""
|
||||
result = CriteriaResult(
|
||||
criteria="Be helpful",
|
||||
scoring_type="numeric",
|
||||
threshold=7.0,
|
||||
eval_id="crit123"
|
||||
)
|
||||
d = result.to_dict()
|
||||
assert d["criteria"] == "Be helpful"
|
||||
assert d["scoring_type"] == "numeric"
|
||||
assert d["threshold"] == 7.0
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Unit tests for evaluation utility functions."""
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from praisonaiagents.eval.utils import (
|
||||
save_result_to_file,
|
||||
load_result_from_file,
|
||||
format_score,
|
||||
format_percentage,
|
||||
format_duration,
|
||||
format_memory,
|
||||
calculate_pass_rate,
|
||||
)
|
||||
|
||||
|
||||
class TestSaveResultToFile:
|
||||
"""Tests for save_result_to_file function."""
|
||||
|
||||
def test_save_with_to_dict(self):
|
||||
"""Test saving result with to_dict method."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/result.json"
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0, "passed": True}
|
||||
|
||||
success = save_result_to_file(result, path)
|
||||
|
||||
assert success is True
|
||||
assert Path(path).exists()
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
assert data["score"] == 8.0
|
||||
|
||||
def test_save_with_dict_attr(self):
|
||||
"""Test saving result with __dict__ attribute."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/result.json"
|
||||
|
||||
class SimpleResult:
|
||||
def __init__(self):
|
||||
self.score = 9.0
|
||||
self.name = "test"
|
||||
|
||||
result = SimpleResult()
|
||||
success = save_result_to_file(result, path)
|
||||
|
||||
assert success is True
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
assert data["score"] == 9.0
|
||||
|
||||
def test_save_with_placeholders(self):
|
||||
"""Test saving with path placeholders."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/{{name}}_{{eval_id}}.json"
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
success = save_result_to_file(
|
||||
result, path,
|
||||
eval_id="abc123",
|
||||
name="test_eval"
|
||||
)
|
||||
|
||||
assert success is True
|
||||
expected_path = f"{tmpdir}/test_eval_abc123.json"
|
||||
assert Path(expected_path).exists()
|
||||
|
||||
def test_save_creates_directories(self):
|
||||
"""Test that save creates parent directories."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/nested/dir/result.json"
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
success = save_result_to_file(result, path)
|
||||
|
||||
assert success is True
|
||||
assert Path(path).exists()
|
||||
|
||||
def test_save_failure_returns_false(self):
|
||||
"""Test that save failure returns False."""
|
||||
result = MagicMock()
|
||||
result.to_dict.side_effect = Exception("Test error")
|
||||
|
||||
success = save_result_to_file(result, "/invalid/path/result.json")
|
||||
|
||||
assert success is False
|
||||
|
||||
|
||||
class TestLoadResultFromFile:
|
||||
"""Tests for load_result_from_file function."""
|
||||
|
||||
def test_load_existing_file(self):
|
||||
"""Test loading existing result file."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/result.json"
|
||||
data = {"score": 8.0, "passed": True}
|
||||
with open(path, 'w') as f:
|
||||
json.dump(data, f)
|
||||
|
||||
loaded = load_result_from_file(path)
|
||||
|
||||
assert loaded == data
|
||||
|
||||
def test_load_nonexistent_file(self):
|
||||
"""Test loading nonexistent file returns None."""
|
||||
result = load_result_from_file("/nonexistent/path/result.json")
|
||||
assert result is None
|
||||
|
||||
def test_load_invalid_json(self):
|
||||
"""Test loading invalid JSON returns None."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/invalid.json"
|
||||
with open(path, 'w') as f:
|
||||
f.write("not valid json")
|
||||
|
||||
result = load_result_from_file(path)
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestFormatScore:
|
||||
"""Tests for format_score function."""
|
||||
|
||||
def test_format_default_max(self):
|
||||
"""Test formatting with default max score."""
|
||||
assert format_score(8.5) == "8.5/10"
|
||||
|
||||
def test_format_custom_max(self):
|
||||
"""Test formatting with custom max score."""
|
||||
assert format_score(85.0, 100.0) == "85.0/100"
|
||||
|
||||
def test_format_zero(self):
|
||||
"""Test formatting zero score."""
|
||||
assert format_score(0.0) == "0.0/10"
|
||||
|
||||
|
||||
class TestFormatPercentage:
|
||||
"""Tests for format_percentage function."""
|
||||
|
||||
def test_format_full(self):
|
||||
"""Test formatting 100%."""
|
||||
assert format_percentage(1.0) == "100.0%"
|
||||
|
||||
def test_format_half(self):
|
||||
"""Test formatting 50%."""
|
||||
assert format_percentage(0.5) == "50.0%"
|
||||
|
||||
def test_format_zero(self):
|
||||
"""Test formatting 0%."""
|
||||
assert format_percentage(0.0) == "0.0%"
|
||||
|
||||
def test_format_decimal(self):
|
||||
"""Test formatting decimal percentage."""
|
||||
assert format_percentage(0.333) == "33.3%"
|
||||
|
||||
|
||||
class TestFormatDuration:
|
||||
"""Tests for format_duration function."""
|
||||
|
||||
def test_format_microseconds(self):
|
||||
"""Test formatting microseconds."""
|
||||
result = format_duration(0.0001)
|
||||
assert "μs" in result
|
||||
|
||||
def test_format_milliseconds(self):
|
||||
"""Test formatting milliseconds."""
|
||||
result = format_duration(0.5)
|
||||
assert "ms" in result
|
||||
|
||||
def test_format_seconds(self):
|
||||
"""Test formatting seconds."""
|
||||
result = format_duration(5.5)
|
||||
assert "s" in result
|
||||
assert "m" not in result
|
||||
|
||||
def test_format_minutes(self):
|
||||
"""Test formatting minutes."""
|
||||
result = format_duration(90.0)
|
||||
assert "m" in result
|
||||
|
||||
|
||||
class TestFormatMemory:
|
||||
"""Tests for format_memory function."""
|
||||
|
||||
def test_format_kilobytes(self):
|
||||
"""Test formatting kilobytes."""
|
||||
result = format_memory(0.5)
|
||||
assert "KB" in result
|
||||
|
||||
def test_format_megabytes(self):
|
||||
"""Test formatting megabytes."""
|
||||
result = format_memory(100.0)
|
||||
assert "MB" in result
|
||||
|
||||
def test_format_gigabytes(self):
|
||||
"""Test formatting gigabytes."""
|
||||
result = format_memory(2048.0)
|
||||
assert "GB" in result
|
||||
|
||||
|
||||
class TestCalculatePassRate:
|
||||
"""Tests for calculate_pass_rate function."""
|
||||
|
||||
def test_all_passed(self):
|
||||
"""Test 100% pass rate."""
|
||||
assert calculate_pass_rate(10, 10) == 1.0
|
||||
|
||||
def test_none_passed(self):
|
||||
"""Test 0% pass rate."""
|
||||
assert calculate_pass_rate(0, 10) == 0.0
|
||||
|
||||
def test_half_passed(self):
|
||||
"""Test 50% pass rate."""
|
||||
assert calculate_pass_rate(5, 10) == 0.5
|
||||
|
||||
def test_zero_total(self):
|
||||
"""Test with zero total."""
|
||||
assert calculate_pass_rate(0, 0) == 0.0
|
||||
@@ -0,0 +1 @@
|
||||
"""Unit tests for Agent Skills module."""
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Tests for Agent Skills discovery."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestGetDefaultSkillDirs:
|
||||
"""Tests for get_default_skill_dirs function."""
|
||||
|
||||
def test_returns_list(self):
|
||||
"""Test that function returns a list."""
|
||||
from praisonaiagents.skills.discovery import get_default_skill_dirs
|
||||
|
||||
result = get_default_skill_dirs()
|
||||
|
||||
assert isinstance(result, list)
|
||||
|
||||
def test_returns_path_objects(self):
|
||||
"""Test that returned items are Path objects."""
|
||||
from praisonaiagents.skills.discovery import get_default_skill_dirs
|
||||
|
||||
result = get_default_skill_dirs()
|
||||
|
||||
for item in result:
|
||||
assert isinstance(item, Path)
|
||||
|
||||
|
||||
class TestDiscoverSkills:
|
||||
"""Tests for discover_skills function."""
|
||||
|
||||
def test_discover_empty_directory(self):
|
||||
"""Test discovering skills in empty directory."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
result = discover_skills([tmpdir], include_defaults=False)
|
||||
|
||||
assert result == []
|
||||
|
||||
def test_discover_single_skill(self):
|
||||
"""Test discovering a single valid skill."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create a skill directory
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill for discovery
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
""")
|
||||
|
||||
result = discover_skills([tmpdir], include_defaults=False)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].name == "test-skill"
|
||||
assert result[0].description == "A test skill for discovery"
|
||||
|
||||
def test_discover_multiple_skills(self):
|
||||
"""Test discovering multiple skills."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create first skill
|
||||
skill1_dir = Path(tmpdir) / "skill-one"
|
||||
skill1_dir.mkdir()
|
||||
(skill1_dir / "SKILL.md").write_text("""---
|
||||
name: skill-one
|
||||
description: First skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create second skill
|
||||
skill2_dir = Path(tmpdir) / "skill-two"
|
||||
skill2_dir.mkdir()
|
||||
(skill2_dir / "SKILL.md").write_text("""---
|
||||
name: skill-two
|
||||
description: Second skill
|
||||
---
|
||||
""")
|
||||
|
||||
result = discover_skills([tmpdir], include_defaults=False)
|
||||
|
||||
assert len(result) == 2
|
||||
names = {s.name for s in result}
|
||||
assert "skill-one" in names
|
||||
assert "skill-two" in names
|
||||
|
||||
def test_discover_skips_invalid_skills(self):
|
||||
"""Test that invalid skills are skipped."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create valid skill
|
||||
valid_dir = Path(tmpdir) / "valid-skill"
|
||||
valid_dir.mkdir()
|
||||
(valid_dir / "SKILL.md").write_text("""---
|
||||
name: valid-skill
|
||||
description: A valid skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create invalid skill (no SKILL.md)
|
||||
invalid_dir = Path(tmpdir) / "invalid-skill"
|
||||
invalid_dir.mkdir()
|
||||
|
||||
result = discover_skills([tmpdir], include_defaults=False)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].name == "valid-skill"
|
||||
|
||||
def test_discover_from_multiple_directories(self):
|
||||
"""Test discovering from multiple directories."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir1:
|
||||
with tempfile.TemporaryDirectory() as tmpdir2:
|
||||
# Create skill in first directory
|
||||
skill1_dir = Path(tmpdir1) / "skill-a"
|
||||
skill1_dir.mkdir()
|
||||
(skill1_dir / "SKILL.md").write_text("""---
|
||||
name: skill-a
|
||||
description: Skill A
|
||||
---
|
||||
""")
|
||||
|
||||
# Create skill in second directory
|
||||
skill2_dir = Path(tmpdir2) / "skill-b"
|
||||
skill2_dir.mkdir()
|
||||
(skill2_dir / "SKILL.md").write_text("""---
|
||||
name: skill-b
|
||||
description: Skill B
|
||||
---
|
||||
""")
|
||||
|
||||
result = discover_skills([tmpdir1, tmpdir2], include_defaults=False)
|
||||
|
||||
assert len(result) == 2
|
||||
|
||||
|
||||
class TestDiscoverSkill:
|
||||
"""Tests for discover_skill function (single skill)."""
|
||||
|
||||
def test_discover_valid_skill(self):
|
||||
"""Test discovering a single valid skill."""
|
||||
from praisonaiagents.skills.discovery import discover_skill
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "my-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: my-skill
|
||||
description: My skill
|
||||
---
|
||||
""")
|
||||
|
||||
result = discover_skill(str(skill_dir))
|
||||
|
||||
assert result is not None
|
||||
assert result.name == "my-skill"
|
||||
|
||||
def test_discover_nonexistent_path(self):
|
||||
"""Test discovering from nonexistent path."""
|
||||
from praisonaiagents.skills.discovery import discover_skill
|
||||
|
||||
result = discover_skill("/nonexistent/path")
|
||||
|
||||
assert result is None
|
||||
|
||||
def test_discover_directory_without_skill_md(self):
|
||||
"""Test discovering from directory without SKILL.md."""
|
||||
from praisonaiagents.skills.discovery import discover_skill
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
result = discover_skill(tmpdir)
|
||||
|
||||
assert result is None
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Tests for Agent Skills loader."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestSkillLoader:
|
||||
"""Tests for SkillLoader class."""
|
||||
|
||||
def test_load_metadata_valid_skill(self):
|
||||
"""Test loading metadata from valid skill."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
# Instructions
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
|
||||
assert skill is not None
|
||||
assert skill.properties.name == "test-skill"
|
||||
assert skill.instructions is None # Not activated yet
|
||||
|
||||
def test_load_metadata_nonexistent(self):
|
||||
"""Test loading from nonexistent path."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata("/nonexistent/path")
|
||||
|
||||
assert skill is None
|
||||
|
||||
def test_activate_skill(self):
|
||||
"""Test activating a skill loads instructions."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
# Instructions
|
||||
|
||||
Follow these steps:
|
||||
1. Do this
|
||||
2. Do that
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
|
||||
assert not skill.is_activated
|
||||
|
||||
result = loader.activate(skill)
|
||||
|
||||
assert result is True
|
||||
assert skill.is_activated
|
||||
assert "Follow these steps" in skill.instructions
|
||||
|
||||
def test_activate_already_activated(self):
|
||||
"""Test activating already activated skill returns True."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
Instructions.
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
loader.activate(skill)
|
||||
|
||||
# Activate again
|
||||
result = loader.activate(skill)
|
||||
|
||||
assert result is True
|
||||
|
||||
def test_load_scripts(self):
|
||||
"""Test loading scripts from skill."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create scripts directory
|
||||
scripts_dir = skill_dir / "scripts"
|
||||
scripts_dir.mkdir()
|
||||
(scripts_dir / "helper.py").write_text("print('hello')")
|
||||
(scripts_dir / "utils.sh").write_text("echo 'hello'")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
scripts = loader.load_scripts(skill)
|
||||
|
||||
assert "helper.py" in scripts
|
||||
assert "utils.sh" in scripts
|
||||
assert "print('hello')" in scripts["helper.py"]
|
||||
|
||||
def test_load_references(self):
|
||||
"""Test loading references from skill."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create references directory
|
||||
refs_dir = skill_dir / "references"
|
||||
refs_dir.mkdir()
|
||||
(refs_dir / "api.md").write_text("# API Reference")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
refs = loader.load_references(skill)
|
||||
|
||||
assert "api.md" in refs
|
||||
assert "# API Reference" in refs["api.md"]
|
||||
|
||||
def test_load_assets(self):
|
||||
"""Test loading assets from skill."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create assets directory
|
||||
assets_dir = skill_dir / "assets"
|
||||
assets_dir.mkdir()
|
||||
(assets_dir / "template.json").write_text("{}")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
assets = loader.load_assets(skill)
|
||||
|
||||
assert "template.json" in assets
|
||||
# Path may be resolved differently on macOS (/var vs /private/var)
|
||||
assert assets["template.json"].endswith("test-skill/assets/template.json")
|
||||
|
||||
def test_class_method_load(self):
|
||||
"""Test convenience class method load."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
Instructions here.
|
||||
""")
|
||||
|
||||
skill = SkillLoader.load(str(skill_dir), activate=True)
|
||||
|
||||
assert skill is not None
|
||||
assert skill.is_activated
|
||||
assert "Instructions here" in skill.instructions
|
||||
|
||||
|
||||
class TestLoadedSkill:
|
||||
"""Tests for LoadedSkill dataclass."""
|
||||
|
||||
def test_metadata_property(self):
|
||||
"""Test metadata property returns SkillMetadata."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
|
||||
meta = skill.metadata
|
||||
|
||||
assert isinstance(meta, SkillMetadata)
|
||||
assert meta.name == "test-skill"
|
||||
assert meta.description == "A test skill"
|
||||
|
||||
def test_is_activated_property(self):
|
||||
"""Test is_activated property."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
Body.
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
|
||||
assert skill.is_activated is False
|
||||
|
||||
loader.activate(skill)
|
||||
|
||||
assert skill.is_activated is True
|
||||
@@ -0,0 +1,254 @@
|
||||
"""Tests for Agent Skills manager."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestSkillManager:
|
||||
"""Tests for SkillManager class."""
|
||||
|
||||
def test_init_empty(self):
|
||||
"""Test initializing empty manager."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
manager = SkillManager()
|
||||
|
||||
assert len(manager) == 0
|
||||
assert manager.skill_names == []
|
||||
|
||||
def test_discover_skills(self):
|
||||
"""Test discovering skills from directory."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create a skill
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
count = manager.discover([tmpdir], include_defaults=False)
|
||||
|
||||
assert count == 1
|
||||
assert len(manager) == 1
|
||||
assert "test-skill" in manager
|
||||
|
||||
def test_add_skill(self):
|
||||
"""Test adding a single skill."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "my-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: my-skill
|
||||
description: My skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
skill = manager.add_skill(str(skill_dir))
|
||||
|
||||
assert skill is not None
|
||||
assert skill.properties.name == "my-skill"
|
||||
assert "my-skill" in manager
|
||||
|
||||
def test_get_skill(self):
|
||||
"""Test getting a skill by name."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
skill = manager.get_skill("test-skill")
|
||||
|
||||
assert skill is not None
|
||||
assert skill.properties.name == "test-skill"
|
||||
|
||||
def test_get_skill_not_found(self):
|
||||
"""Test getting nonexistent skill."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
manager = SkillManager()
|
||||
skill = manager.get_skill("nonexistent")
|
||||
|
||||
assert skill is None
|
||||
|
||||
def test_activate_by_name(self):
|
||||
"""Test activating skill by name."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
# Instructions
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
result = manager.activate_by_name("test-skill")
|
||||
|
||||
assert result is True
|
||||
skill = manager.get_skill("test-skill")
|
||||
assert skill.is_activated
|
||||
|
||||
def test_get_available_skills(self):
|
||||
"""Test getting available skills metadata."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
skills = manager.get_available_skills()
|
||||
|
||||
assert len(skills) == 1
|
||||
assert isinstance(skills[0], SkillMetadata)
|
||||
assert skills[0].name == "test-skill"
|
||||
|
||||
def test_to_prompt(self):
|
||||
"""Test generating prompt XML."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
prompt = manager.to_prompt()
|
||||
|
||||
assert "<available_skills>" in prompt
|
||||
assert "test-skill" in prompt
|
||||
assert "A test skill" in prompt
|
||||
|
||||
def test_get_instructions(self):
|
||||
"""Test getting instructions for a skill."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
# Instructions
|
||||
|
||||
Do this and that.
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
instructions = manager.get_instructions("test-skill")
|
||||
|
||||
assert instructions is not None
|
||||
assert "Do this and that" in instructions
|
||||
|
||||
def test_load_resources(self):
|
||||
"""Test loading resources for a skill."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create scripts directory
|
||||
scripts_dir = skill_dir / "scripts"
|
||||
scripts_dir.mkdir()
|
||||
(scripts_dir / "helper.py").write_text("print('hello')")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
result = manager.load_resources("test-skill")
|
||||
|
||||
assert result is True
|
||||
skill = manager.get_skill("test-skill")
|
||||
assert skill.resources_loaded
|
||||
|
||||
def test_clear(self):
|
||||
"""Test clearing all skills."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
assert len(manager) == 1
|
||||
|
||||
manager.clear()
|
||||
|
||||
assert len(manager) == 0
|
||||
|
||||
def test_iteration(self):
|
||||
"""Test iterating over skills."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create two skills
|
||||
for name in ["skill-a", "skill-b"]:
|
||||
skill_dir = Path(tmpdir) / name
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text(f"""---
|
||||
name: {name}
|
||||
description: Skill {name}
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.discover([tmpdir], include_defaults=False)
|
||||
|
||||
names = [s.properties.name for s in manager]
|
||||
|
||||
assert len(names) == 2
|
||||
assert "skill-a" in names
|
||||
assert "skill-b" in names
|
||||
@@ -0,0 +1,184 @@
|
||||
"""Tests for Agent Skills models."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class TestSkillProperties:
|
||||
"""Tests for SkillProperties dataclass."""
|
||||
|
||||
def test_skill_properties_required_fields(self):
|
||||
"""Test that SkillProperties requires name and description."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill for unit testing"
|
||||
)
|
||||
|
||||
assert skill.name == "test-skill"
|
||||
assert skill.description == "A test skill for unit testing"
|
||||
|
||||
def test_skill_properties_optional_fields_default_none(self):
|
||||
"""Test that optional fields default to None."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill"
|
||||
)
|
||||
|
||||
assert skill.license is None
|
||||
assert skill.compatibility is None
|
||||
assert skill.allowed_tools is None
|
||||
assert skill.path is None
|
||||
|
||||
def test_skill_properties_metadata_defaults_empty_dict(self):
|
||||
"""Test that metadata defaults to empty dict."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill"
|
||||
)
|
||||
|
||||
assert skill.metadata == {}
|
||||
|
||||
def test_skill_properties_all_fields(self):
|
||||
"""Test SkillProperties with all fields populated."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="pdf-processing",
|
||||
description="Extract text and tables from PDF files",
|
||||
license="Apache-2.0",
|
||||
compatibility="Requires pypdf package",
|
||||
allowed_tools="Read Grep Bash",
|
||||
metadata={"author": "test-org", "version": "1.0"},
|
||||
path=Path("/path/to/skill")
|
||||
)
|
||||
|
||||
assert skill.name == "pdf-processing"
|
||||
assert skill.description == "Extract text and tables from PDF files"
|
||||
assert skill.license == "Apache-2.0"
|
||||
assert skill.compatibility == "Requires pypdf package"
|
||||
assert skill.allowed_tools == "Read Grep Bash"
|
||||
assert skill.metadata == {"author": "test-org", "version": "1.0"}
|
||||
assert skill.path == Path("/path/to/skill")
|
||||
|
||||
def test_skill_properties_to_dict(self):
|
||||
"""Test to_dict method excludes None values."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
license="MIT"
|
||||
)
|
||||
|
||||
result = skill.to_dict()
|
||||
|
||||
assert result["name"] == "test-skill"
|
||||
assert result["description"] == "A test skill"
|
||||
assert result["license"] == "MIT"
|
||||
assert "compatibility" not in result
|
||||
assert "allowed-tools" not in result
|
||||
assert "metadata" not in result # Empty metadata excluded
|
||||
|
||||
def test_skill_properties_to_dict_with_metadata(self):
|
||||
"""Test to_dict includes non-empty metadata."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
metadata={"version": "1.0"}
|
||||
)
|
||||
|
||||
result = skill.to_dict()
|
||||
|
||||
assert result["metadata"] == {"version": "1.0"}
|
||||
|
||||
def test_skill_properties_to_dict_allowed_tools_key(self):
|
||||
"""Test to_dict uses 'allowed-tools' key (hyphenated)."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
allowed_tools="Read Write"
|
||||
)
|
||||
|
||||
result = skill.to_dict()
|
||||
|
||||
assert "allowed-tools" in result
|
||||
assert result["allowed-tools"] == "Read Write"
|
||||
|
||||
|
||||
class TestSkillMetadata:
|
||||
"""Tests for SkillMetadata dataclass (lightweight for system prompt)."""
|
||||
|
||||
def test_skill_metadata_required_fields(self):
|
||||
"""Test SkillMetadata with required fields."""
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
meta = SkillMetadata(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
location="/path/to/skill/SKILL.md"
|
||||
)
|
||||
|
||||
assert meta.name == "test-skill"
|
||||
assert meta.description == "A test skill"
|
||||
assert meta.location == "/path/to/skill/SKILL.md"
|
||||
|
||||
def test_skill_metadata_from_properties(self):
|
||||
"""Test creating SkillMetadata from SkillProperties."""
|
||||
from praisonaiagents.skills.models import SkillProperties, SkillMetadata
|
||||
|
||||
props = SkillProperties(
|
||||
name="pdf-skill",
|
||||
description="Process PDF files",
|
||||
path=Path("/skills/pdf-skill")
|
||||
)
|
||||
|
||||
meta = SkillMetadata.from_properties(props)
|
||||
|
||||
assert meta.name == "pdf-skill"
|
||||
assert meta.description == "Process PDF files"
|
||||
assert "/skills/pdf-skill" in meta.location
|
||||
|
||||
|
||||
class TestParseError:
|
||||
"""Tests for ParseError exception."""
|
||||
|
||||
def test_parse_error_message(self):
|
||||
"""Test ParseError stores message correctly."""
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
error = ParseError("Invalid YAML frontmatter")
|
||||
|
||||
assert str(error) == "Invalid YAML frontmatter"
|
||||
|
||||
def test_parse_error_is_exception(self):
|
||||
"""Test ParseError is an Exception subclass."""
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
assert issubclass(ParseError, Exception)
|
||||
|
||||
|
||||
class TestValidationError:
|
||||
"""Tests for ValidationError exception."""
|
||||
|
||||
def test_validation_error_message(self):
|
||||
"""Test ValidationError stores message correctly."""
|
||||
from praisonaiagents.skills.models import ValidationError
|
||||
|
||||
error = ValidationError("Missing required field: name")
|
||||
|
||||
assert str(error) == "Missing required field: name"
|
||||
|
||||
def test_validation_error_is_exception(self):
|
||||
"""Test ValidationError is an Exception subclass."""
|
||||
from praisonaiagents.skills.models import ValidationError
|
||||
|
||||
assert issubclass(ValidationError, Exception)
|
||||
@@ -0,0 +1,288 @@
|
||||
"""Tests for Agent Skills parser."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestParseFrontmatter:
|
||||
"""Tests for parse_frontmatter function."""
|
||||
|
||||
def test_parse_valid_frontmatter(self):
|
||||
"""Test parsing valid YAML frontmatter."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
|
||||
content = """---
|
||||
name: test-skill
|
||||
description: A test skill for unit testing
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
|
||||
Instructions go here.
|
||||
"""
|
||||
metadata, body = parse_frontmatter(content)
|
||||
|
||||
assert metadata["name"] == "test-skill"
|
||||
assert metadata["description"] == "A test skill for unit testing"
|
||||
assert "# Test Skill" in body
|
||||
|
||||
def test_parse_frontmatter_with_optional_fields(self):
|
||||
"""Test parsing frontmatter with all optional fields."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
|
||||
content = """---
|
||||
name: pdf-processing
|
||||
description: Extract text from PDFs
|
||||
license: Apache-2.0
|
||||
compatibility: Requires pypdf
|
||||
allowed-tools: Read Grep
|
||||
metadata:
|
||||
author: test-org
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
Body content.
|
||||
"""
|
||||
metadata, body = parse_frontmatter(content)
|
||||
|
||||
assert metadata["name"] == "pdf-processing"
|
||||
assert metadata["license"] == "Apache-2.0"
|
||||
assert metadata["compatibility"] == "Requires pypdf"
|
||||
assert metadata["allowed-tools"] == "Read Grep"
|
||||
assert metadata["metadata"]["author"] == "test-org"
|
||||
assert metadata["metadata"]["version"] == "1.0"
|
||||
|
||||
def test_parse_frontmatter_missing_opening_delimiter(self):
|
||||
"""Test error when frontmatter doesn't start with ---."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
content = """name: test-skill
|
||||
description: Missing opening delimiter
|
||||
---
|
||||
|
||||
Body content.
|
||||
"""
|
||||
try:
|
||||
parse_frontmatter(content)
|
||||
assert False, "Should have raised ParseError"
|
||||
except ParseError as e:
|
||||
assert "must start with" in str(e).lower() or "---" in str(e)
|
||||
|
||||
def test_parse_frontmatter_missing_closing_delimiter(self):
|
||||
"""Test error when frontmatter is not closed."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
content = """---
|
||||
name: test-skill
|
||||
description: Missing closing delimiter
|
||||
|
||||
Body content without closing delimiter.
|
||||
"""
|
||||
try:
|
||||
parse_frontmatter(content)
|
||||
assert False, "Should have raised ParseError"
|
||||
except ParseError as e:
|
||||
assert "closed" in str(e).lower() or "---" in str(e)
|
||||
|
||||
def test_parse_frontmatter_invalid_yaml(self):
|
||||
"""Test error when YAML is invalid."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
content = """---
|
||||
name: test-skill
|
||||
description: [invalid yaml
|
||||
- missing bracket
|
||||
---
|
||||
|
||||
Body content.
|
||||
"""
|
||||
try:
|
||||
parse_frontmatter(content)
|
||||
assert False, "Should have raised ParseError"
|
||||
except ParseError as e:
|
||||
assert "yaml" in str(e).lower() or "invalid" in str(e).lower()
|
||||
|
||||
def test_parse_frontmatter_empty_body(self):
|
||||
"""Test parsing when body is empty."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
|
||||
content = """---
|
||||
name: test-skill
|
||||
description: A skill with no body
|
||||
---
|
||||
"""
|
||||
metadata, body = parse_frontmatter(content)
|
||||
|
||||
assert metadata["name"] == "test-skill"
|
||||
assert body.strip() == ""
|
||||
|
||||
|
||||
class TestFindSkillMd:
|
||||
"""Tests for find_skill_md function."""
|
||||
|
||||
def test_find_skill_md_uppercase(self):
|
||||
"""Test finding SKILL.md (uppercase)."""
|
||||
from praisonaiagents.skills.parser import find_skill_md
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir)
|
||||
skill_md = skill_dir / "SKILL.md"
|
||||
skill_md.write_text("---\nname: test\ndescription: test\n---\n")
|
||||
|
||||
result = find_skill_md(skill_dir)
|
||||
|
||||
assert result is not None
|
||||
assert result.name == "SKILL.md"
|
||||
|
||||
def test_find_skill_md_lowercase(self):
|
||||
"""Test finding skill.md (lowercase) as fallback."""
|
||||
from praisonaiagents.skills.parser import find_skill_md
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir)
|
||||
skill_md = skill_dir / "skill.md"
|
||||
skill_md.write_text("---\nname: test\ndescription: test\n---\n")
|
||||
|
||||
result = find_skill_md(skill_dir)
|
||||
|
||||
assert result is not None
|
||||
# On case-insensitive filesystems (macOS), SKILL.md may match skill.md
|
||||
assert result.name.lower() == "skill.md"
|
||||
|
||||
def test_find_skill_md_prefers_uppercase(self):
|
||||
"""Test that SKILL.md is preferred over skill.md."""
|
||||
from praisonaiagents.skills.parser import find_skill_md
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir)
|
||||
(skill_dir / "SKILL.md").write_text("uppercase")
|
||||
(skill_dir / "skill.md").write_text("lowercase")
|
||||
|
||||
result = find_skill_md(skill_dir)
|
||||
|
||||
assert result is not None
|
||||
assert result.name == "SKILL.md"
|
||||
|
||||
def test_find_skill_md_not_found(self):
|
||||
"""Test returning None when no SKILL.md exists."""
|
||||
from praisonaiagents.skills.parser import find_skill_md
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir)
|
||||
|
||||
result = find_skill_md(skill_dir)
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestReadProperties:
|
||||
"""Tests for read_properties function."""
|
||||
|
||||
def test_read_properties_valid_skill(self):
|
||||
"""Test reading properties from a valid skill directory."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill for unit testing
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
""")
|
||||
|
||||
props = read_properties(skill_dir)
|
||||
|
||||
assert props.name == "test-skill"
|
||||
assert props.description == "A test skill for unit testing"
|
||||
|
||||
def test_read_properties_with_all_fields(self):
|
||||
"""Test reading properties with all optional fields."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "pdf-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: pdf-skill
|
||||
description: Process PDF files
|
||||
license: MIT
|
||||
compatibility: Requires pypdf
|
||||
allowed-tools: Read Write
|
||||
metadata:
|
||||
author: test
|
||||
version: "2.0"
|
||||
---
|
||||
|
||||
Instructions.
|
||||
""")
|
||||
|
||||
props = read_properties(skill_dir)
|
||||
|
||||
assert props.name == "pdf-skill"
|
||||
assert props.license == "MIT"
|
||||
assert props.compatibility == "Requires pypdf"
|
||||
assert props.allowed_tools == "Read Write"
|
||||
assert props.metadata["author"] == "test"
|
||||
|
||||
def test_read_properties_missing_skill_md(self):
|
||||
"""Test error when SKILL.md is missing."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "empty-skill"
|
||||
skill_dir.mkdir()
|
||||
|
||||
try:
|
||||
read_properties(skill_dir)
|
||||
assert False, "Should have raised ParseError"
|
||||
except ParseError as e:
|
||||
assert "not found" in str(e).lower() or "skill.md" in str(e).lower()
|
||||
|
||||
def test_read_properties_missing_name(self):
|
||||
"""Test error when name field is missing."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
from praisonaiagents.skills.models import ValidationError
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "no-name"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
description: Missing name field
|
||||
---
|
||||
|
||||
Body.
|
||||
""")
|
||||
|
||||
try:
|
||||
read_properties(skill_dir)
|
||||
assert False, "Should have raised ValidationError"
|
||||
except ValidationError as e:
|
||||
assert "name" in str(e).lower()
|
||||
|
||||
def test_read_properties_missing_description(self):
|
||||
"""Test error when description field is missing."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
from praisonaiagents.skills.models import ValidationError
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "no-desc"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: no-desc
|
||||
---
|
||||
|
||||
Body.
|
||||
""")
|
||||
|
||||
try:
|
||||
read_properties(skill_dir)
|
||||
assert False, "Should have raised ValidationError"
|
||||
except ValidationError as e:
|
||||
assert "description" in str(e).lower()
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Tests for Agent Skills prompt generation."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestToPrompt:
|
||||
"""Tests for to_prompt function (XML generation)."""
|
||||
|
||||
def test_to_prompt_empty_list(self):
|
||||
"""Test generating prompt with no skills."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
result = to_prompt([])
|
||||
|
||||
assert "<available_skills>" in result
|
||||
assert "</available_skills>" in result
|
||||
|
||||
def test_to_prompt_single_skill(self):
|
||||
"""Test generating prompt with one skill."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill for prompt generation
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
""")
|
||||
|
||||
result = to_prompt([skill_dir])
|
||||
|
||||
assert "<available_skills>" in result
|
||||
assert "</available_skills>" in result
|
||||
assert "<skill>" in result
|
||||
assert "</skill>" in result
|
||||
assert "<name>" in result
|
||||
assert "test-skill" in result
|
||||
assert "<description>" in result
|
||||
assert "A test skill for prompt generation" in result
|
||||
assert "<location>" in result
|
||||
|
||||
def test_to_prompt_multiple_skills(self):
|
||||
"""Test generating prompt with multiple skills."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create first skill
|
||||
skill1_dir = Path(tmpdir) / "skill-one"
|
||||
skill1_dir.mkdir()
|
||||
(skill1_dir / "SKILL.md").write_text("""---
|
||||
name: skill-one
|
||||
description: First skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create second skill
|
||||
skill2_dir = Path(tmpdir) / "skill-two"
|
||||
skill2_dir.mkdir()
|
||||
(skill2_dir / "SKILL.md").write_text("""---
|
||||
name: skill-two
|
||||
description: Second skill
|
||||
---
|
||||
""")
|
||||
|
||||
result = to_prompt([skill1_dir, skill2_dir])
|
||||
|
||||
assert result.count("<skill>") == 2
|
||||
assert result.count("</skill>") == 2
|
||||
assert "skill-one" in result
|
||||
assert "skill-two" in result
|
||||
assert "First skill" in result
|
||||
assert "Second skill" in result
|
||||
|
||||
def test_to_prompt_escapes_html(self):
|
||||
"""Test that special characters are HTML escaped."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "escape-test"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: escape-test
|
||||
description: Test <script> & "quotes" handling
|
||||
---
|
||||
""")
|
||||
|
||||
result = to_prompt([skill_dir])
|
||||
|
||||
# Should escape < > & "
|
||||
assert "<script>" not in result or "<script>" in result
|
||||
assert "&" in result or "& " not in result
|
||||
|
||||
def test_to_prompt_includes_location(self):
|
||||
"""Test that location points to SKILL.md file."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "location-test"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: location-test
|
||||
description: Test location field
|
||||
---
|
||||
""")
|
||||
|
||||
result = to_prompt([skill_dir])
|
||||
|
||||
assert "<location>" in result
|
||||
assert "SKILL.md" in result
|
||||
assert "</location>" in result
|
||||
|
||||
|
||||
class TestGenerateSkillsXml:
|
||||
"""Tests for generate_skills_xml helper function."""
|
||||
|
||||
def test_generate_skills_xml_from_metadata_list(self):
|
||||
"""Test generating XML from SkillMetadata list."""
|
||||
from praisonaiagents.skills.prompt import generate_skills_xml
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
skills = [
|
||||
SkillMetadata(
|
||||
name="pdf-skill",
|
||||
description="Process PDF files",
|
||||
location="/path/to/pdf-skill/SKILL.md"
|
||||
),
|
||||
SkillMetadata(
|
||||
name="excel-skill",
|
||||
description="Process Excel files",
|
||||
location="/path/to/excel-skill/SKILL.md"
|
||||
)
|
||||
]
|
||||
|
||||
result = generate_skills_xml(skills)
|
||||
|
||||
assert "<available_skills>" in result
|
||||
assert "pdf-skill" in result
|
||||
assert "excel-skill" in result
|
||||
assert "Process PDF files" in result
|
||||
assert "Process Excel files" in result
|
||||
|
||||
def test_generate_skills_xml_empty_list(self):
|
||||
"""Test generating XML with empty list."""
|
||||
from praisonaiagents.skills.prompt import generate_skills_xml
|
||||
|
||||
result = generate_skills_xml([])
|
||||
|
||||
assert "<available_skills>" in result
|
||||
assert "</available_skills>" in result
|
||||
assert "<skill>" not in result
|
||||
|
||||
|
||||
class TestFormatSkillForPrompt:
|
||||
"""Tests for format_skill_for_prompt helper."""
|
||||
|
||||
def test_format_single_skill(self):
|
||||
"""Test formatting a single skill for prompt."""
|
||||
from praisonaiagents.skills.prompt import format_skill_for_prompt
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
skill = SkillMetadata(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
location="/path/to/SKILL.md"
|
||||
)
|
||||
|
||||
result = format_skill_for_prompt(skill)
|
||||
|
||||
assert "<skill>" in result
|
||||
assert "</skill>" in result
|
||||
assert "<name>" in result
|
||||
assert "test-skill" in result
|
||||
assert "<description>" in result
|
||||
assert "A test skill" in result
|
||||
assert "<location>" in result
|
||||
assert "/path/to/SKILL.md" in result
|
||||
@@ -0,0 +1,258 @@
|
||||
"""Tests for Agent Skills validator."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestValidateName:
|
||||
"""Tests for skill name validation."""
|
||||
|
||||
def test_valid_name_simple(self):
|
||||
"""Test valid simple skill name."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("test-skill", None)
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_valid_name_with_numbers(self):
|
||||
"""Test valid name with numbers."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("skill-v2", None)
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_invalid_name_uppercase(self):
|
||||
"""Test that uppercase letters are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("Test-Skill", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("lowercase" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_starts_with_hyphen(self):
|
||||
"""Test that names starting with hyphen are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("-skill", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("hyphen" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_ends_with_hyphen(self):
|
||||
"""Test that names ending with hyphen are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("skill-", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("hyphen" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_consecutive_hyphens(self):
|
||||
"""Test that consecutive hyphens are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("test--skill", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("consecutive" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_too_long(self):
|
||||
"""Test that names over 64 characters are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
long_name = "a" * 65
|
||||
errors = _validate_name(long_name, None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("64" in e or "limit" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_special_characters(self):
|
||||
"""Test that special characters are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("test_skill", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("invalid" in e.lower() or "character" in e.lower() for e in errors)
|
||||
|
||||
def test_name_must_match_directory(self):
|
||||
"""Test that name must match parent directory name."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "actual-name"
|
||||
skill_dir.mkdir()
|
||||
|
||||
errors = _validate_name("different-name", skill_dir)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("match" in e.lower() for e in errors)
|
||||
|
||||
|
||||
class TestValidateDescription:
|
||||
"""Tests for skill description validation."""
|
||||
|
||||
def test_valid_description(self):
|
||||
"""Test valid description."""
|
||||
from praisonaiagents.skills.validator import _validate_description
|
||||
|
||||
errors = _validate_description("A helpful skill that does things")
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_invalid_description_empty(self):
|
||||
"""Test that empty description is rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_description
|
||||
|
||||
errors = _validate_description("")
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("empty" in e.lower() or "non-empty" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_description_too_long(self):
|
||||
"""Test that descriptions over 1024 characters are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_description
|
||||
|
||||
long_desc = "a" * 1025
|
||||
errors = _validate_description(long_desc)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("1024" in e or "limit" in e.lower() for e in errors)
|
||||
|
||||
|
||||
class TestValidateCompatibility:
|
||||
"""Tests for skill compatibility validation."""
|
||||
|
||||
def test_valid_compatibility(self):
|
||||
"""Test valid compatibility string."""
|
||||
from praisonaiagents.skills.validator import _validate_compatibility
|
||||
|
||||
errors = _validate_compatibility("Requires Python 3.11+")
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_invalid_compatibility_too_long(self):
|
||||
"""Test that compatibility over 500 characters is rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_compatibility
|
||||
|
||||
long_compat = "a" * 501
|
||||
errors = _validate_compatibility(long_compat)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("500" in e or "limit" in e.lower() for e in errors)
|
||||
|
||||
|
||||
class TestValidateMetadata:
|
||||
"""Tests for full metadata validation."""
|
||||
|
||||
def test_valid_metadata(self):
|
||||
"""Test valid complete metadata."""
|
||||
from praisonaiagents.skills.validator import validate_metadata
|
||||
|
||||
metadata = {
|
||||
"name": "test-skill",
|
||||
"description": "A test skill"
|
||||
}
|
||||
|
||||
errors = validate_metadata(metadata)
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_missing_name(self):
|
||||
"""Test error when name is missing."""
|
||||
from praisonaiagents.skills.validator import validate_metadata
|
||||
|
||||
metadata = {
|
||||
"description": "A test skill"
|
||||
}
|
||||
|
||||
errors = validate_metadata(metadata)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("name" in e.lower() for e in errors)
|
||||
|
||||
def test_missing_description(self):
|
||||
"""Test error when description is missing."""
|
||||
from praisonaiagents.skills.validator import validate_metadata
|
||||
|
||||
metadata = {
|
||||
"name": "test-skill"
|
||||
}
|
||||
|
||||
errors = validate_metadata(metadata)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("description" in e.lower() for e in errors)
|
||||
|
||||
def test_unexpected_fields_rejected(self):
|
||||
"""Test that unexpected fields are rejected."""
|
||||
from praisonaiagents.skills.validator import validate_metadata
|
||||
|
||||
metadata = {
|
||||
"name": "test-skill",
|
||||
"description": "A test skill",
|
||||
"unknown-field": "value"
|
||||
}
|
||||
|
||||
errors = validate_metadata(metadata)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("unexpected" in e.lower() or "unknown-field" in e for e in errors)
|
||||
|
||||
|
||||
class TestValidate:
|
||||
"""Tests for full skill directory validation."""
|
||||
|
||||
def test_validate_valid_skill(self):
|
||||
"""Test validating a valid skill directory."""
|
||||
from praisonaiagents.skills.validator import validate
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill for validation
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
""")
|
||||
|
||||
errors = validate(skill_dir)
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_validate_missing_skill_md(self):
|
||||
"""Test error when SKILL.md is missing."""
|
||||
from praisonaiagents.skills.validator import validate
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "empty-skill"
|
||||
skill_dir.mkdir()
|
||||
|
||||
errors = validate(skill_dir)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("skill.md" in e.lower() for e in errors)
|
||||
|
||||
def test_validate_nonexistent_directory(self):
|
||||
"""Test error when directory doesn't exist."""
|
||||
from praisonaiagents.skills.validator import validate
|
||||
|
||||
errors = validate(Path("/nonexistent/path"))
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("exist" in e.lower() for e in errors)
|
||||
|
||||
def test_validate_not_a_directory(self):
|
||||
"""Test error when path is not a directory."""
|
||||
from praisonaiagents.skills.validator import validate
|
||||
|
||||
with tempfile.NamedTemporaryFile() as tmpfile:
|
||||
errors = validate(Path(tmpfile.name))
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("directory" in e.lower() for e in errors)
|
||||
Generated
+1
-1
@@ -2422,7 +2422,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "praisonaiagents"
|
||||
version = "0.1.21"
|
||||
version = "0.1.23"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "aiohttp" },
|
||||
|
||||
+99
-122
@@ -171,12 +171,13 @@ npm install praisonai
|
||||
|
||||
| Feature | Code | Docs |
|
||||
|---------|:----:|:----:|
|
||||
| 🚀 Single Agent | [Example](examples/python/agents/single-agent.py) | [📖](https://docs.praison.ai/agents/single) |
|
||||
| 🤝 Multi Agents | [Example](examples/python/general/mini_agents_example.py) | [📖](https://docs.praison.ai/concepts/agents) |
|
||||
| 🤖 Auto Agents | [Example](examples/python/general/auto_agents_example.py) | [📖](https://docs.praison.ai/features/autoagents) |
|
||||
| 🔄 Self Reflection AI Agents | [Example](examples/python/concepts/self-reflection-details.py) | [📖](https://docs.praison.ai/features/selfreflection) |
|
||||
| 🧠 Reasoning AI Agents | [Example](examples/python/concepts/reasoning-extraction.py) | [📖](https://docs.praison.ai/features/reasoning) |
|
||||
| 👁️ Multi Modal AI Agents | [Example](examples/python/general/multimodal.py) | [📖](https://docs.praison.ai/features/multimodal) |
|
||||
| **🤖 Core Agents** | | |
|
||||
| ↳ Single Agent | [Example](examples/python/agents/single-agent.py) | [📖](https://docs.praison.ai/agents/single) |
|
||||
| ↳ Multi Agents | [Example](examples/python/general/mini_agents_example.py) | [📖](https://docs.praison.ai/concepts/agents) |
|
||||
| ↳ Auto Agents | [Example](examples/python/general/auto_agents_example.py) | [📖](https://docs.praison.ai/features/autoagents) |
|
||||
| ↳ Self Reflection AI Agents | [Example](examples/python/concepts/self-reflection-details.py) | [📖](https://docs.praison.ai/features/selfreflection) |
|
||||
| ↳ Reasoning AI Agents | [Example](examples/python/concepts/reasoning-extraction.py) | [📖](https://docs.praison.ai/features/reasoning) |
|
||||
| ↳ Multi Modal AI Agents | [Example](examples/python/general/multimodal.py) | [📖](https://docs.praison.ai/features/multimodal) |
|
||||
| **🔄 Workflows** | | |
|
||||
| ↳ Simple Workflow | [Example](examples/python/workflows/simple_workflow.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow with Agents | [Example](examples/python/workflows/workflow_with_agents.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
@@ -188,10 +189,7 @@ npm install praisonai
|
||||
| ↳ Workflow Branching | [Example](examples/python/workflows/workflow_branching.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow Early Stop | [Example](examples/python/workflows/workflow_early_stop.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow Checkpoints | [Example](examples/python/workflows/workflow_checkpoints.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| 📚 Add Custom Knowledge | [Example](examples/python/concepts/knowledge-agents.py) | [📖](https://docs.praison.ai/features/knowledge) |
|
||||
| 🧠 Memory (Short & Long Term) | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| 📄 Chat with PDF Agents | [Example](examples/python/concepts/chat-with-pdf.py) | [📖](https://docs.praison.ai/features/chat-with-pdf) |
|
||||
| **💻 Code** | | |
|
||||
| **💻 Code & Development** | | |
|
||||
| ↳ Code Interpreter Agents | [Example](examples/python/agents/code-agent.py) | [📖](https://docs.praison.ai/features/codeagent) |
|
||||
| ↳ AI Code Editing Tools | [Example](examples/python/code/code_editing_example.py) | [📖](https://docs.praison.ai/code/editing) |
|
||||
| ↳ External Agents (All) | [Example](examples/python/code/external_agents_example.py) | [📖](https://docs.praison.ai/code/external-agents) |
|
||||
@@ -199,65 +197,78 @@ npm install praisonai
|
||||
| ↳ Gemini CLI | [Example](examples/python/code/gemini_cli_example.py) | [📖](https://docs.praison.ai/code/gemini-cli) |
|
||||
| ↳ Codex CLI | [Example](examples/python/code/codex_cli_example.py) | [📖](https://docs.praison.ai/code/codex-cli) |
|
||||
| ↳ Cursor CLI | [Example](examples/python/code/cursor_cli_example.py) | [📖](https://docs.praison.ai/code/cursor-cli) |
|
||||
| 📚 RAG Agents | [Example](examples/python/concepts/rag-agents.py) | [📖](https://docs.praison.ai/features/rag) |
|
||||
| 🤔 Async & Parallel Processing | [Example](examples/python/general/async_example.py) | [📖](https://docs.praison.ai/features/async) |
|
||||
| 🔢 Math Agents | [Example](examples/python/agents/math-agent.py) | [📖](https://docs.praison.ai/features/mathagent) |
|
||||
| 🎯 Structured Output Agents | [Example](examples/python/general/structured_agents_example.py) | [📖](https://docs.praison.ai/features/structured) |
|
||||
| 🔗 LangChain Integrated Agents | [Example](examples/python/general/langchain_example.py) | [📖](https://docs.praison.ai/features/langchain) |
|
||||
| 📞 Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
|
||||
| 🛠️ 100+ Custom Tools | [Example](examples/python/general/tools_example.py) | [📖](https://docs.praison.ai/tools/tools) |
|
||||
| 📄 YAML Configuration | [Example](examples/cookbooks/yaml/secondary_market_research_agents.yaml) | [📖](https://docs.praison.ai/developers/agents-playbook) |
|
||||
| 💯 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
|
||||
| 🔬 Deep Research Agents | [Example](examples/python/agents/research-agent.py) | [📖](https://docs.praison.ai/agents/deep-research) |
|
||||
| 🔄 Query Rewriter Agent | [Example](#5-query-rewriter-agent) | [📖](https://docs.praison.ai/agents/query-rewriter) |
|
||||
| 🌐 Native Web Search | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/agents/websearch) |
|
||||
| 📥 Web Fetch (Anthropic) | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| 💾 Prompt Caching | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| 🧠 Claude Memory Tool | [Example](#claude-memory-tool-cli) | [📖](https://docs.praison.ai/features/claude-memory-tool) |
|
||||
| 💾 File-Based Memory | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| 🔍 Built-in Search Tools | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/tools/tavily) |
|
||||
| 🔎 Unified Web Search | [Example](src/praisonai-agents/examples/web_search_example.py) | [📖](https://docs.praison.ai/tools/web-search) |
|
||||
| 📋 Planning Mode | [Example](examples/python/agents/planning-agent.py) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🔧 Planning Tools | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🧠 Planning Reasoning | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🔌 MCP Transports | [Example](examples/python/mcp/mcp-transports-overview.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| 🌐 WebSocket MCP | [Example](examples/python/mcp/websocket-mcp.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| 🔐 MCP Security | [Example](examples/python/mcp/mcp-security.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| 🔄 MCP Resumability | [Example](examples/python/mcp/mcp-resumability.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ⚡ Fast Context | [Example](examples/context/00_agent_fast_context_basic.py) | [📖](https://docs.praison.ai/features/fast-context) |
|
||||
| 🖼️ Image Generation Agent | [Example](examples/python/image/image-agent.py) | [📖](https://docs.praison.ai/features/image-generation) |
|
||||
| 📷 Image to Text Agent | [Example](examples/python/agents/image-to-text-agent.py) | [📖](https://docs.praison.ai/agents/image-to-text) |
|
||||
| 🎬 Video Agent | [Example](examples/python/agents/video-agent.py) | [📖](https://docs.praison.ai/agents/video) |
|
||||
| 📊 Data Analyst Agent | [Example](examples/python/agents/data-analyst-agent.py) | [📖](https://docs.praison.ai/agents/data-analyst) |
|
||||
| 💰 Finance Agent | [Example](examples/python/agents/finance-agent.py) | [📖](https://docs.praison.ai/agents/finance) |
|
||||
| 🛒 Shopping Agent | [Example](examples/python/agents/shopping-agent.py) | [📖](https://docs.praison.ai/agents/shopping) |
|
||||
| ⭐ Recommendation Agent | [Example](examples/python/agents/recommendation-agent.py) | [📖](https://docs.praison.ai/agents/recommendation) |
|
||||
| 📖 Wikipedia Agent | [Example](examples/python/agents/wikipedia-agent.py) | [📖](https://docs.praison.ai/agents/wikipedia) |
|
||||
| 💻 Programming Agent | [Example](examples/python/agents/programming-agent.py) | [📖](https://docs.praison.ai/agents/programming) |
|
||||
| 📝 Markdown Agent | [Example](examples/python/agents/markdown-agent.py) | [📖](https://docs.praison.ai/agents/markdown) |
|
||||
| 📝 Prompt Expander Agent | [Example](#prompt-expansion) | [📖](https://docs.praison.ai/agents/prompt-expander) |
|
||||
| 🔀 Model Router | [Example](examples/python/agents/router-agent-cost-optimization.py) | [📖](https://docs.praison.ai/features/model-router) |
|
||||
| ⛓️ Prompt Chaining | [Example](examples/python/general/prompt_chaining.py) | [📖](https://docs.praison.ai/features/promptchaining) |
|
||||
| 🔍 Evaluator Optimiser | [Example](examples/python/general/evaluator-optimiser.py) | [📖](https://docs.praison.ai/features/evaluator-optimiser) |
|
||||
| 👷 Orchestrator Workers | [Example](examples/python/general/orchestrator-workers.py) | [📖](https://docs.praison.ai/features/orchestrator-worker) |
|
||||
| ⚡ Parallelisation | [Example](examples/python/general/parallelisation.py) | [📖](https://docs.praison.ai/features/parallelisation) |
|
||||
| 🔁 Repetitive Agents | [Example](examples/python/concepts/repetitive-agents.py) | [📖](https://docs.praison.ai/features/repetitive) |
|
||||
| 🤝 Agent Handoffs | [Example](examples/python/handoff/handoff_basic.py) | [📖](https://docs.praison.ai/features/handoffs) |
|
||||
| 🛡️ Guardrails | [Example](examples/python/guardrails/comprehensive-guardrails-example.py) | [📖](https://docs.praison.ai/features/guardrails) |
|
||||
| 💬 Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
|
||||
| ✅ Human Approval | [Example](examples/python/general/human_approval_example.py) | [📖](https://docs.praison.ai/features/approval) |
|
||||
| 🔄 Stateful Agents | [Example](examples/python/stateful/workflow-state-example.py) | [📖](https://docs.praison.ai/features/stateful-agents) |
|
||||
| 🤖 Autonomous Workflow | [Example](examples/python/general/autonomous-agent.py) | [📖](https://docs.praison.ai/features/autonomous-workflow) |
|
||||
| 📜 Rules & Instructions | [Example](#6-rules--instructions) | [📖](https://docs.praison.ai/features/rules) |
|
||||
| 🪝 Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
|
||||
| 📈 Telemetry | [Example](examples/python/telemetry/production-telemetry-example.py) | [📖](https://docs.praison.ai/features/telemetry) |
|
||||
| 📹 Camera Integration | [Example](examples/python/camera/) | [📖](https://docs.praison.ai/features/camera-integration) |
|
||||
| 📄 Project Docs (.praison/docs/) | [Example](#docs-cli) | [📖](https://docs.praison.ai/docs/cli/docs) |
|
||||
| 🔌 MCP Config Management | [Example](#mcp-config-cli) | [📖](https://docs.praison.ai/docs/cli/mcp) |
|
||||
| 💬 AI Commit Messages | [Example](#ai-commit-cli) | [📖](https://docs.praison.ai/docs/cli/commit) |
|
||||
| @ @Mentions in Prompts | [Example](#mentions-in-prompts) | [📖](https://docs.praison.ai/docs/cli/mentions) |
|
||||
| 💾 Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| 📜 History in Context | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| **🧠 Memory & Knowledge** | | |
|
||||
| ↳ Memory (Short & Long Term) | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| ↳ File-Based Memory | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| ↳ Claude Memory Tool | [Example](#claude-memory-tool-cli) | [📖](https://docs.praison.ai/features/claude-memory-tool) |
|
||||
| ↳ Add Custom Knowledge | [Example](examples/python/concepts/knowledge-agents.py) | [📖](https://docs.praison.ai/features/knowledge) |
|
||||
| ↳ RAG Agents | [Example](examples/python/concepts/rag-agents.py) | [📖](https://docs.praison.ai/features/rag) |
|
||||
| ↳ Chat with PDF Agents | [Example](examples/python/concepts/chat-with-pdf.py) | [📖](https://docs.praison.ai/features/chat-with-pdf) |
|
||||
| **🔬 Research & Intelligence** | | |
|
||||
| ↳ Deep Research Agents | [Example](examples/python/agents/research-agent.py) | [📖](https://docs.praison.ai/agents/deep-research) |
|
||||
| ↳ Query Rewriter Agent | [Example](#5-query-rewriter-agent) | [📖](https://docs.praison.ai/agents/query-rewriter) |
|
||||
| ↳ Native Web Search | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/agents/websearch) |
|
||||
| ↳ Built-in Search Tools | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/tools/tavily) |
|
||||
| ↳ Unified Web Search | [Example](src/praisonai-agents/examples/web_search_example.py) | [📖](https://docs.praison.ai/tools/web-search) |
|
||||
| ↳ Web Fetch (Anthropic) | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| **📋 Planning & Execution** | | |
|
||||
| ↳ Planning Mode | [Example](examples/python/agents/planning-agent.py) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Planning Tools | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Planning Reasoning | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Prompt Chaining | [Example](examples/python/general/prompt_chaining.py) | [📖](https://docs.praison.ai/features/promptchaining) |
|
||||
| ↳ Evaluator Optimiser | [Example](examples/python/general/evaluator-optimiser.py) | [📖](https://docs.praison.ai/features/evaluator-optimiser) |
|
||||
| ↳ Orchestrator Workers | [Example](examples/python/general/orchestrator-workers.py) | [📖](https://docs.praison.ai/features/orchestrator-worker) |
|
||||
| **👥 Specialized Agents** | | |
|
||||
| ↳ Data Analyst Agent | [Example](examples/python/agents/data-analyst-agent.py) | [📖](https://docs.praison.ai/agents/data-analyst) |
|
||||
| ↳ Finance Agent | [Example](examples/python/agents/finance-agent.py) | [📖](https://docs.praison.ai/agents/finance) |
|
||||
| ↳ Shopping Agent | [Example](examples/python/agents/shopping-agent.py) | [📖](https://docs.praison.ai/agents/shopping) |
|
||||
| ↳ Recommendation Agent | [Example](examples/python/agents/recommendation-agent.py) | [📖](https://docs.praison.ai/agents/recommendation) |
|
||||
| ↳ Wikipedia Agent | [Example](examples/python/agents/wikipedia-agent.py) | [📖](https://docs.praison.ai/agents/wikipedia) |
|
||||
| ↳ Programming Agent | [Example](examples/python/agents/programming-agent.py) | [📖](https://docs.praison.ai/agents/programming) |
|
||||
| ↳ Math Agents | [Example](examples/python/agents/math-agent.py) | [📖](https://docs.praison.ai/features/mathagent) |
|
||||
| ↳ Markdown Agent | [Example](examples/python/agents/markdown-agent.py) | [📖](https://docs.praison.ai/agents/markdown) |
|
||||
| ↳ Prompt Expander Agent | [Example](#prompt-expansion) | [📖](https://docs.praison.ai/agents/prompt-expander) |
|
||||
| **🎨 Media & Multimodal** | | |
|
||||
| ↳ Image Generation Agent | [Example](examples/python/image/image-agent.py) | [📖](https://docs.praison.ai/features/image-generation) |
|
||||
| ↳ Image to Text Agent | [Example](examples/python/agents/image-to-text-agent.py) | [📖](https://docs.praison.ai/agents/image-to-text) |
|
||||
| ↳ Video Agent | [Example](examples/python/agents/video-agent.py) | [📖](https://docs.praison.ai/agents/video) |
|
||||
| ↳ Camera Integration | [Example](examples/python/camera/) | [📖](https://docs.praison.ai/features/camera-integration) |
|
||||
| **🔌 Protocols & Integration** | | |
|
||||
| ↳ MCP Transports | [Example](examples/python/mcp/mcp-transports-overview.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| ↳ WebSocket MCP | [Example](examples/python/mcp/websocket-mcp.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ↳ MCP Security | [Example](examples/python/mcp/mcp-security.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| ↳ MCP Resumability | [Example](examples/python/mcp/mcp-resumability.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ↳ MCP Config Management | [Example](#mcp-config-cli) | [📖](https://docs.praison.ai/docs/cli/mcp) |
|
||||
| ↳ LangChain Integrated Agents | [Example](examples/python/general/langchain_example.py) | [📖](https://docs.praison.ai/features/langchain) |
|
||||
| **🛡️ Safety & Control** | | |
|
||||
| ↳ Guardrails | [Example](examples/python/guardrails/comprehensive-guardrails-example.py) | [📖](https://docs.praison.ai/features/guardrails) |
|
||||
| ↳ Human Approval | [Example](examples/python/general/human_approval_example.py) | [📖](https://docs.praison.ai/features/approval) |
|
||||
| ↳ Rules & Instructions | [Example](#6-rules--instructions) | [📖](https://docs.praison.ai/features/rules) |
|
||||
| **⚙️ Advanced Features** | | |
|
||||
| ↳ Async & Parallel Processing | [Example](examples/python/general/async_example.py) | [📖](https://docs.praison.ai/features/async) |
|
||||
| ↳ Parallelisation | [Example](examples/python/general/parallelisation.py) | [📖](https://docs.praison.ai/features/parallelisation) |
|
||||
| ↳ Repetitive Agents | [Example](examples/python/concepts/repetitive-agents.py) | [📖](https://docs.praison.ai/features/repetitive) |
|
||||
| ↳ Agent Handoffs | [Example](examples/python/handoff/handoff_basic.py) | [📖](https://docs.praison.ai/features/handoffs) |
|
||||
| ↳ Stateful Agents | [Example](examples/python/stateful/workflow-state-example.py) | [📖](https://docs.praison.ai/features/stateful-agents) |
|
||||
| ↳ Autonomous Workflow | [Example](examples/python/general/autonomous-agent.py) | [📖](https://docs.praison.ai/features/autonomous-workflow) |
|
||||
| ↳ Structured Output Agents | [Example](examples/python/general/structured_agents_example.py) | [📖](https://docs.praison.ai/features/structured) |
|
||||
| ↳ Model Router | [Example](examples/python/agents/router-agent-cost-optimization.py) | [📖](https://docs.praison.ai/features/model-router) |
|
||||
| ↳ Prompt Caching | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| ↳ Fast Context | [Example](examples/context/00_agent_fast_context_basic.py) | [📖](https://docs.praison.ai/features/fast-context) |
|
||||
| **🛠️ Tools & Configuration** | | |
|
||||
| ↳ 100+ Custom Tools | [Example](examples/python/general/tools_example.py) | [📖](https://docs.praison.ai/tools/tools) |
|
||||
| ↳ YAML Configuration | [Example](examples/cookbooks/yaml/secondary_market_research_agents.yaml) | [📖](https://docs.praison.ai/developers/agents-playbook) |
|
||||
| ↳ 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
|
||||
| ↳ Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
|
||||
| ↳ Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
|
||||
| **📊 Monitoring & Management** | | |
|
||||
| ↳ Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
|
||||
| ↳ Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| ↳ History in Context | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| ↳ Telemetry | [Example](examples/python/telemetry/production-telemetry-example.py) | [📖](https://docs.praison.ai/features/telemetry) |
|
||||
| ↳ Project Docs (.praison/docs/) | [Example](#docs-cli) | [📖](https://docs.praison.ai/docs/cli/docs) |
|
||||
| ↳ AI Commit Messages | [Example](#ai-commit-cli) | [📖](https://docs.praison.ai/docs/cli/commit) |
|
||||
| ↳ @Mentions in Prompts | [Example](#mentions-in-prompts) | [📖](https://docs.praison.ai/docs/cli/mentions) |
|
||||
| **🖥️ CLI Features** | | |
|
||||
| ↳ Slash Commands | [Example](examples/python/cli/slash_commands_example.py) | [📖](https://docs.praison.ai/docs/cli/slash-commands) |
|
||||
| ↳ Autonomy Modes | [Example](examples/python/cli/autonomy_modes_example.py) | [📖](https://docs.praison.ai/docs/cli/autonomy-modes) |
|
||||
@@ -266,6 +277,15 @@ npm install praisonai
|
||||
| ↳ Interactive TUI | [Example](examples/python/cli/interactive_tui_example.py) | [📖](https://docs.praison.ai/docs/cli/interactive-tui) |
|
||||
| ↳ Git Integration | [Example](examples/python/cli/git_integration_example.py) | [📖](https://docs.praison.ai/docs/cli/git-integration) |
|
||||
| ↳ Sandbox Execution | [Example](examples/python/cli/sandbox_execution_example.py) | [📖](https://docs.praison.ai/docs/cli/sandbox-execution) |
|
||||
| ↳ CLI Compare | [Example](examples/compare/cli_compare_basic.py) | [📖](https://docs.praison.ai/docs/cli/compare) |
|
||||
| **🧪 Evaluation** | | |
|
||||
| ↳ Accuracy Evaluation | [Example](examples/eval/accuracy_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Performance Evaluation | [Example](examples/eval/performance_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Reliability Evaluation | [Example](examples/eval/reliability_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Criteria Evaluation | [Example](examples/eval/criteria_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| **🎯 Agent Skills** | | |
|
||||
| ↳ Skills Management | [Example](examples/skills/basic_skill_usage.py) | [📖](https://docs.praison.ai/features/skills) |
|
||||
| ↳ Custom Skills | [Example](examples/skills/custom_skill_example.py) | [📖](https://docs.praison.ai/features/skills) |
|
||||
| **⏰ 24/7 Scheduling** | | |
|
||||
| ↳ Agent Scheduler | [Example](examples/python/scheduled_agents/news_checker_live.py) | [📖](https://docs.praison.ai/docs/cli/scheduler) |
|
||||
|
||||
@@ -1246,62 +1266,19 @@ praisonai "Complex task" --metrics --planning
|
||||
|
||||
### Scheduler CLI:
|
||||
|
||||
Run agents 24/7 with built-in safety features. Two modes available:
|
||||
|
||||
**Direct Prompt Mode (Simple):**
|
||||
```bash
|
||||
# No YAML needed - just provide your task
|
||||
praisonai schedule "Check AI news and summarize" --interval hourly
|
||||
|
||||
# With options
|
||||
praisonai schedule "Monitor server logs" \
|
||||
--interval "*/30m" \
|
||||
--timeout 120 \
|
||||
--max-cost 2.00 \
|
||||
--verbose
|
||||
praisonai schedule start <name> "task" --interval hourly
|
||||
praisonai schedule list
|
||||
praisonai schedule logs <name> [--follow]
|
||||
praisonai schedule stop <name>
|
||||
praisonai schedule restart <name>
|
||||
praisonai schedule delete <name>
|
||||
praisonai schedule describe <name>
|
||||
praisonai schedule save <name> [file.yaml]
|
||||
praisonai schedule "task" --interval hourly # foreground mode
|
||||
praisonai schedule agents.yaml # foreground mode
|
||||
```
|
||||
|
||||
**YAML Mode (Advanced):**
|
||||
```bash
|
||||
# Use agents.yaml for complex configurations
|
||||
praisonai schedule agents.yaml
|
||||
|
||||
# With overrides
|
||||
praisonai schedule agents.yaml \
|
||||
--interval "*/15m" \
|
||||
--timeout 60 \
|
||||
--max-cost 1.00
|
||||
|
||||
# Stop: Press Ctrl+C
|
||||
```
|
||||
|
||||
**agents.yaml with schedule:**
|
||||
```yaml
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "AI News Monitor"
|
||||
instructions: "Search and summarize AI news"
|
||||
tools:
|
||||
- search_tool
|
||||
|
||||
task: "Search for latest AI news"
|
||||
|
||||
schedule:
|
||||
interval: "hourly" # hourly, daily, */30m, */6h
|
||||
max_retries: 3 # Retry on failure
|
||||
run_immediately: true # Run once on start
|
||||
timeout: 60 # Max 60s per execution
|
||||
max_cost: 1.00 # Stop after $1.00 spent
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- ⏱️ Timeout protection (prevent runaway executions)
|
||||
- 💰 Cost monitoring with budget limits
|
||||
- 📊 Real-time statistics (success rate, cost, runtime)
|
||||
- 🔄 Automatic retry with exponential backoff
|
||||
- 🛡️ Auto-stops when budget reached
|
||||
|
||||
### Image Processing CLI:
|
||||
```bash
|
||||
# Process images with vision-based tasks
|
||||
|
||||
@@ -3,8 +3,8 @@ class Praisonai < Formula
|
||||
|
||||
desc "AI tools for various AI applications"
|
||||
homepage "https://github.com/MervinPraison/PraisonAI"
|
||||
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.67.tar.gz"
|
||||
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.67.tar.gz | shasum -a 256`.split.first
|
||||
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.75.tar.gz"
|
||||
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.75.tar.gz | shasum -a 256`.split.first
|
||||
license "MIT"
|
||||
|
||||
depends_on "[email protected]"
|
||||
|
||||
@@ -27,6 +27,8 @@ Features:
|
||||
- cost_tracker: Real-time cost and token tracking
|
||||
- message_queue: Message queuing while agent is processing
|
||||
- at_mentions: @ mention autocomplete for files and directories
|
||||
- compare: CLI command comparison feature
|
||||
- eval: Agent evaluation framework (accuracy, performance, reliability, criteria)
|
||||
"""
|
||||
|
||||
# Type hints available for IDE support
|
||||
@@ -63,6 +65,12 @@ __all__ = [
|
||||
'AtMentionCompleter',
|
||||
'CombinedCompleter',
|
||||
'FileSearchService',
|
||||
# Evaluation
|
||||
'EvalHandler',
|
||||
# Compare feature
|
||||
'CompareHandler',
|
||||
# Agent Skills
|
||||
'SkillsHandler',
|
||||
]
|
||||
|
||||
def __getattr__(name):
|
||||
@@ -153,4 +161,13 @@ def __getattr__(name):
|
||||
elif name == 'FileSearchService':
|
||||
from .at_mentions import FileSearchService
|
||||
return FileSearchService
|
||||
elif name == 'CompareHandler':
|
||||
from .compare import CompareHandler
|
||||
return CompareHandler
|
||||
elif name == 'EvalHandler':
|
||||
from .eval import EvalHandler
|
||||
return EvalHandler
|
||||
elif name == 'SkillsHandler':
|
||||
from .skills import SkillsHandler
|
||||
return SkillsHandler
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -43,8 +43,18 @@ class AgentSchedulerHandler:
|
||||
return AgentSchedulerHandler._handle_restart(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "delete":
|
||||
return AgentSchedulerHandler._handle_delete(unknown_args, state_manager)
|
||||
elif subcommand == "describe":
|
||||
return AgentSchedulerHandler._handle_describe(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "save":
|
||||
return AgentSchedulerHandler._handle_save(unknown_args, state_manager)
|
||||
elif subcommand == "stop-all":
|
||||
return AgentSchedulerHandler._handle_stop_all(state_manager, daemon_manager)
|
||||
elif subcommand == "stats":
|
||||
return AgentSchedulerHandler._handle_stats(state_manager, unknown_args, daemon_manager)
|
||||
else:
|
||||
print(f"Unknown subcommand: {subcommand}")
|
||||
print(f"❌ Unknown subcommand: {subcommand}")
|
||||
print("\nAvailable commands:")
|
||||
print(" start, list, stop, logs, restart, delete, describe, save, stop-all, stats")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
@@ -120,7 +130,8 @@ class AgentSchedulerHandler:
|
||||
print(f" Logs: ~/.praisonai/logs/{name}.log")
|
||||
print(f"\nManage:")
|
||||
print(f" praisonai schedule list")
|
||||
print(f" praisonai schedule logs {name} -f")
|
||||
print(f" praisonai schedule describe {name}")
|
||||
print(f" praisonai schedule logs {name} --follow")
|
||||
print(f" praisonai schedule stop {name}")
|
||||
|
||||
return 0
|
||||
@@ -157,6 +168,94 @@ class AgentSchedulerHandler:
|
||||
print(f"\nTotal: {len(states)} scheduler(s)")
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_stats(state_manager, unknown_args=None, daemon_manager=None) -> int:
|
||||
"""Handle 'schedule stats' command - show aggregate or individual statistics."""
|
||||
# If a name is provided, show individual stats (alias for describe)
|
||||
if unknown_args and len(unknown_args) > 0:
|
||||
name = unknown_args[0]
|
||||
return AgentSchedulerHandler._handle_describe([name], state_manager, daemon_manager)
|
||||
|
||||
# Otherwise show aggregate stats
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers found.")
|
||||
print("\nStart one with:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
return 0
|
||||
|
||||
# Calculate aggregate stats
|
||||
total_schedulers = len(states)
|
||||
running = sum(1 for s in states if s.get('status') == 'running')
|
||||
stopped = total_schedulers - running
|
||||
total_executions = sum(s.get('executions', 0) for s in states)
|
||||
total_cost = sum(s.get('cost', 0.0) for s in states)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"📊 Aggregate Scheduler Statistics")
|
||||
print(f"{'='*60}")
|
||||
print(f"Total Schedulers: {total_schedulers}")
|
||||
print(f" 🟢 Running: {running}")
|
||||
print(f" 🔴 Stopped: {stopped}")
|
||||
print(f"\nTotal Executions: {total_executions}")
|
||||
print(f"Total Cost: ${total_cost:.4f}")
|
||||
|
||||
if total_executions > 0:
|
||||
avg_cost = total_cost / total_executions
|
||||
print(f"Avg Cost/Execution: ${avg_cost:.4f}")
|
||||
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Show per-scheduler breakdown
|
||||
print("Per-Scheduler Breakdown:")
|
||||
print(f"{'Name':<20} {'Status':<10} {'Executions':<12} {'Cost':<12}")
|
||||
print("="*60)
|
||||
|
||||
for state in sorted(states, key=lambda x: x.get('executions', 0), reverse=True):
|
||||
name = state['name'][:20]
|
||||
status = state.get('status', 'unknown')
|
||||
executions = state.get('executions', 0)
|
||||
cost = state.get('cost', 0.0)
|
||||
|
||||
status_icon = "🟢" if status == "running" else "🔴"
|
||||
print(f"{name:<20} {status_icon} {status:<8} {executions:<12} ${cost:<11.4f}")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_stop_all(state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule stop-all' command."""
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers running.")
|
||||
return 0
|
||||
|
||||
print(f"\n🛑 Stopping all schedulers ({len(states)} total)...\n")
|
||||
|
||||
stopped = 0
|
||||
failed = 0
|
||||
|
||||
for state in states:
|
||||
name = state['name']
|
||||
pid = state['pid']
|
||||
|
||||
try:
|
||||
if daemon_manager.stop_daemon(pid):
|
||||
state_manager.delete_state(name)
|
||||
print(f"✅ Stopped '{name}' (PID: {pid})")
|
||||
stopped += 1
|
||||
else:
|
||||
print(f"❌ Failed to stop '{name}' (PID: {pid})")
|
||||
failed += 1
|
||||
except Exception as e:
|
||||
print(f"❌ Error stopping '{name}': {e}")
|
||||
failed += 1
|
||||
|
||||
print(f"\n📊 Summary: {stopped} stopped, {failed} failed")
|
||||
return 0 if failed == 0 else 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_stop(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule stop' command."""
|
||||
@@ -287,7 +386,129 @@ class AgentSchedulerHandler:
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def handle_schedule_command(args, unknown_args=None) -> int:
|
||||
def _handle_describe(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule describe' command."""
|
||||
from datetime import datetime
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule describe <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
print("\nList schedulers with: praisonai schedule list")
|
||||
return 1
|
||||
|
||||
# Get process status
|
||||
pid = state.get('pid', 0)
|
||||
is_alive = state_manager.is_process_alive(pid)
|
||||
status = "🟢 running" if is_alive else "🔴 stopped"
|
||||
|
||||
# Calculate uptime
|
||||
started_at = state.get('started_at')
|
||||
uptime = "N/A"
|
||||
if started_at:
|
||||
try:
|
||||
start_time = datetime.fromisoformat(started_at)
|
||||
uptime_delta = datetime.now() - start_time
|
||||
hours = int(uptime_delta.total_seconds() // 3600)
|
||||
minutes = int((uptime_delta.total_seconds() % 3600) // 60)
|
||||
uptime = f"{hours}h {minutes}m"
|
||||
except:
|
||||
pass
|
||||
|
||||
# Display detailed info
|
||||
print(f"\n{'='*60}")
|
||||
print(f"📋 Scheduler Details: {name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"Status: {status}")
|
||||
print(f"PID: {pid}")
|
||||
print(f"Uptime: {uptime}")
|
||||
print(f"Task: {state.get('task', 'N/A')}")
|
||||
print(f"Interval: {state.get('interval', 'N/A')}")
|
||||
print(f"Max Retries: {state.get('max_retries', 'N/A')}")
|
||||
|
||||
if state.get('timeout'):
|
||||
print(f"Timeout: {state['timeout']}s")
|
||||
|
||||
if state.get('max_cost'):
|
||||
print(f"Budget: ${state['max_cost']}")
|
||||
|
||||
print(f"Executions: {state.get('executions', 0)}")
|
||||
print(f"Total Cost: ${state.get('cost', 0.0):.4f}")
|
||||
print(f"Started: {started_at or 'N/A'}")
|
||||
|
||||
# Log file location
|
||||
log_file = daemon_manager.log_dir / f"{name}.log"
|
||||
print(f"Logs: {log_file}")
|
||||
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_save(unknown_args, state_manager) -> int:
|
||||
"""Handle 'schedule save' command."""
|
||||
import yaml
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule save <name> [output.yaml]")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
output_file = unknown_args[1] if len(unknown_args) > 1 else f"{name}.yaml"
|
||||
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
# Create YAML config from state
|
||||
yaml_config = {
|
||||
'framework': 'praisonai',
|
||||
'agents': [{
|
||||
'name': name,
|
||||
'role': 'Task Executor',
|
||||
'goal': state.get('task', ''),
|
||||
'instructions': state.get('task', ''),
|
||||
'verbose': True
|
||||
}],
|
||||
'task': state.get('task', ''),
|
||||
'schedule': {
|
||||
'interval': state.get('interval', 'hourly'),
|
||||
'max_retries': state.get('max_retries', 3),
|
||||
'run_immediately': True
|
||||
}
|
||||
}
|
||||
|
||||
# Add optional fields
|
||||
if state.get('timeout'):
|
||||
yaml_config['schedule']['timeout'] = state['timeout']
|
||||
|
||||
if state.get('max_cost'):
|
||||
yaml_config['schedule']['max_cost'] = state['max_cost']
|
||||
|
||||
# Write to file
|
||||
try:
|
||||
with open(output_file, 'w') as f:
|
||||
yaml.dump(yaml_config, f, default_flow_style=False, sort_keys=False)
|
||||
|
||||
print(f"✅ Configuration saved to: {output_file}")
|
||||
print(f"\nRun with:")
|
||||
print(f" praisonai schedule {output_file}")
|
||||
return 0
|
||||
except Exception as e:
|
||||
print(f"❌ Error saving configuration: {e}")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def handle_schedule_command(args, unknown_args, daemon_mode=False) -> int:
|
||||
"""
|
||||
Handle the schedule command for running agents periodically.
|
||||
|
||||
@@ -354,41 +575,33 @@ class AgentSchedulerHandler:
|
||||
print(' praisonai schedule "Check news every hour" --interval hourly')
|
||||
return 1
|
||||
|
||||
task_prompt = first_arg
|
||||
print(f"🤖 Creating scheduler for task: {task_prompt[:60]}...")
|
||||
task = first_arg
|
||||
|
||||
# Create a simple agent with the prompt
|
||||
# Create agent
|
||||
agent = Agent(
|
||||
name="Scheduled Agent",
|
||||
role="Task Executor",
|
||||
goal=task_prompt,
|
||||
instructions=f"Execute this task: {task_prompt}",
|
||||
verbose=verbose
|
||||
goal=task,
|
||||
instructions=task,
|
||||
verbose=True # Enable verbose to see output in logs
|
||||
)
|
||||
|
||||
# Create scheduler
|
||||
scheduler = AgentScheduler(
|
||||
agent=agent,
|
||||
task=task_prompt,
|
||||
task=task,
|
||||
timeout=timeout_override,
|
||||
max_cost=max_cost_override
|
||||
)
|
||||
|
||||
# Display configuration
|
||||
agent_name = getattr(scheduler.agent, 'name', 'Agent')
|
||||
print(f"\n{'='*60}")
|
||||
print(f"🤖 Starting 24/7 {agent_name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"Task: {scheduler.task[:80]}{'...' if len(scheduler.task) > 80 else ''}")
|
||||
# Get configuration
|
||||
interval = interval_override
|
||||
max_retries = max_retries_override
|
||||
|
||||
# Get schedule info
|
||||
if is_yaml_mode:
|
||||
schedule_config = scheduler._yaml_schedule_config
|
||||
interval = interval_override or schedule_config.get('interval', 'hourly')
|
||||
max_retries = max_retries_override or schedule_config.get('max_retries', 3)
|
||||
else:
|
||||
interval = interval_override
|
||||
max_retries = max_retries_override
|
||||
|
||||
print(f"Schedule: {interval}")
|
||||
print(f"Max Retries: {max_retries}")
|
||||
@@ -399,33 +612,38 @@ class AgentSchedulerHandler:
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Start scheduler
|
||||
print("⏰ Starting scheduler... (Press Ctrl+C to stop)\n")
|
||||
if is_yaml_mode:
|
||||
scheduler.start_from_yaml_config()
|
||||
else:
|
||||
scheduler.start(
|
||||
schedule_expr=interval,
|
||||
max_retries=max_retries,
|
||||
run_immediately=True
|
||||
)
|
||||
scheduler.start(schedule_expr=interval, max_retries=max_retries, run_immediately=True)
|
||||
|
||||
# Keep running until interrupted
|
||||
try:
|
||||
while scheduler.is_running:
|
||||
import time
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n🛑 Stopping scheduler...")
|
||||
scheduler.stop()
|
||||
|
||||
# Display final stats
|
||||
stats = scheduler.get_stats()
|
||||
print("\n📊 Final Statistics:")
|
||||
print(f" Total Executions: {stats['total_executions']}")
|
||||
print(f" Successful: {stats['successful_executions']}")
|
||||
print(f" Failed: {stats['failed_executions']}")
|
||||
print(f" Success Rate: {stats['success_rate']:.1f}%")
|
||||
print("\n✅ Agent stopped successfully\n")
|
||||
# If daemon mode, block to keep process alive
|
||||
if daemon_mode:
|
||||
import time
|
||||
try:
|
||||
while scheduler.is_running:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
scheduler.stop()
|
||||
else:
|
||||
# Foreground mode - wait for Ctrl+C
|
||||
print("⏰ Starting scheduler... (Press Ctrl+C to stop)\n")
|
||||
import time
|
||||
try:
|
||||
while scheduler.is_running:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n🛑 Stopping scheduler...")
|
||||
scheduler.stop()
|
||||
|
||||
# Display final stats
|
||||
stats = scheduler.get_stats()
|
||||
print("\n📊 Final Statistics:")
|
||||
print(f" Total Executions: {stats['total_executions']}")
|
||||
print(f" Successful: {stats['successful_executions']}")
|
||||
print(f" Failed: {stats['failed_executions']}")
|
||||
print(f" Success Rate: {stats['success_rate']:.1f}%")
|
||||
print("\n✅ Agent stopped successfully\n")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
@@ -0,0 +1,729 @@
|
||||
"""
|
||||
Agent Scheduler CLI Feature.
|
||||
|
||||
Provides CLI commands for scheduling agents to run 24/7 at regular intervals.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AgentSchedulerHandler:
|
||||
"""Handler for agent scheduler CLI commands."""
|
||||
|
||||
@staticmethod
|
||||
def handle_daemon_command(subcommand: str, args, unknown_args=None) -> int:
|
||||
"""
|
||||
Handle daemon management commands (start, list, stop, logs, restart).
|
||||
|
||||
Args:
|
||||
subcommand: Command to execute (start, list, stop, logs, restart)
|
||||
args: Parsed command-line arguments
|
||||
unknown_args: Additional arguments
|
||||
|
||||
Returns:
|
||||
Exit code
|
||||
"""
|
||||
from praisonai.scheduler.state_manager import SchedulerStateManager
|
||||
from praisonai.scheduler.daemon_manager import DaemonManager
|
||||
|
||||
state_manager = SchedulerStateManager()
|
||||
daemon_manager = DaemonManager()
|
||||
|
||||
if subcommand == "start":
|
||||
return AgentSchedulerHandler._handle_start(args, unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "list":
|
||||
return AgentSchedulerHandler._handle_list(state_manager)
|
||||
elif subcommand == "stop":
|
||||
return AgentSchedulerHandler._handle_stop(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "logs":
|
||||
return AgentSchedulerHandler._handle_logs(unknown_args, daemon_manager)
|
||||
elif subcommand == "restart":
|
||||
return AgentSchedulerHandler._handle_restart(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "delete":
|
||||
return AgentSchedulerHandler._handle_delete(unknown_args, state_manager)
|
||||
elif subcommand == "describe":
|
||||
return AgentSchedulerHandler._handle_describe(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "save":
|
||||
return AgentSchedulerHandler._handle_save(unknown_args, state_manager)
|
||||
elif subcommand == "stop-all":
|
||||
return AgentSchedulerHandler._handle_stop_all(state_manager, daemon_manager)
|
||||
elif subcommand == "stats":
|
||||
return AgentSchedulerHandler._handle_stats(state_manager, unknown_args, daemon_manager)
|
||||
else:
|
||||
print(f"❌ Unknown subcommand: {subcommand}")
|
||||
print("\nAvailable commands:")
|
||||
print(" start, list, stop, logs, restart, delete, describe, save, stop-all, stats")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_start(args, unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule start' command."""
|
||||
from datetime import datetime
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name and task")
|
||||
print("\nUsage:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
print("\nExample:")
|
||||
print(' praisonai schedule start news-checker "Check AI news" --interval hourly')
|
||||
return 1
|
||||
|
||||
# Parse arguments
|
||||
name = unknown_args[0]
|
||||
task = unknown_args[1] if len(unknown_args) > 1 else None
|
||||
|
||||
if not task:
|
||||
print("❌ Error: Please provide a task")
|
||||
return 1
|
||||
|
||||
# Get options
|
||||
interval = getattr(args, 'schedule_interval', None) or 'hourly'
|
||||
max_retries = getattr(args, 'schedule_max_retries', None) or 3
|
||||
timeout = getattr(args, 'timeout', None)
|
||||
max_cost = getattr(args, 'max_cost', None)
|
||||
|
||||
# Check if name already exists
|
||||
existing = state_manager.load_state(name)
|
||||
if existing and daemon_manager.get_status(existing.get('pid', 0))['is_alive']:
|
||||
print(f"❌ Error: Scheduler '{name}' is already running (PID: {existing['pid']})")
|
||||
print(f" Use 'praisonai schedule stop {name}' to stop it first")
|
||||
return 1
|
||||
|
||||
# Start daemon
|
||||
print(f"🚀 Starting scheduler '{name}'...")
|
||||
print(f" Task: {task}")
|
||||
print(f" Interval: {interval}")
|
||||
if timeout:
|
||||
print(f" Timeout: {timeout}s")
|
||||
if max_cost:
|
||||
print(f" Budget: ${max_cost}")
|
||||
|
||||
pid = daemon_manager.start_scheduler_daemon(
|
||||
name=name,
|
||||
task=task,
|
||||
interval=interval,
|
||||
max_cost=max_cost,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries
|
||||
)
|
||||
|
||||
# Save state
|
||||
state = {
|
||||
"name": name,
|
||||
"pid": pid,
|
||||
"task": task,
|
||||
"interval": interval,
|
||||
"timeout": timeout,
|
||||
"max_cost": max_cost,
|
||||
"max_retries": max_retries,
|
||||
"status": "running",
|
||||
"started_at": datetime.now().isoformat(),
|
||||
"executions": 0,
|
||||
"cost": 0.0
|
||||
}
|
||||
state_manager.save_state(name, state)
|
||||
|
||||
print(f"\n✅ Scheduler '{name}' started successfully!")
|
||||
print(f" PID: {pid}")
|
||||
print(f" Logs: ~/.praisonai/logs/{name}.log")
|
||||
print(f"\nManage:")
|
||||
print(f" praisonai schedule list")
|
||||
print(f" praisonai schedule describe {name}")
|
||||
print(f" praisonai schedule logs {name} --follow")
|
||||
print(f" praisonai schedule stop {name}")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_list(state_manager) -> int:
|
||||
"""Handle 'schedule list' command."""
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers running.")
|
||||
print("\nStart one with:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
return 0
|
||||
|
||||
# Clean up dead processes
|
||||
state_manager.cleanup_dead_processes()
|
||||
states = state_manager.list_all()
|
||||
|
||||
# Display table
|
||||
print(f"\n{'Name':<20} {'Status':<10} {'PID':<8} {'Interval':<12} {'Task':<40}")
|
||||
print("=" * 90)
|
||||
|
||||
for state in states:
|
||||
name = state.get('name', 'unknown')[:20]
|
||||
pid = state.get('pid', 0)
|
||||
status = "running" if state_manager.is_process_alive(pid) else "stopped"
|
||||
interval = state.get('interval', 'unknown')[:12]
|
||||
task = state.get('task', '')[:40]
|
||||
|
||||
status_icon = "🟢" if status == "running" else "🔴"
|
||||
print(f"{name:<20} {status_icon} {status:<8} {pid:<8} {interval:<12} {task:<40}")
|
||||
|
||||
print(f"\nTotal: {len(states)} scheduler(s)")
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_stats(state_manager, unknown_args=None, daemon_manager=None) -> int:
|
||||
"""Handle 'schedule stats' command - show aggregate or individual statistics."""
|
||||
# If a name is provided, show individual stats (alias for describe)
|
||||
if unknown_args and len(unknown_args) > 0:
|
||||
name = unknown_args[0]
|
||||
return AgentSchedulerHandler._handle_describe([name], state_manager, None)
|
||||
|
||||
# Otherwise show aggregate stats
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers found.")
|
||||
print("\nStart one with:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
return 0
|
||||
|
||||
# Calculate aggregate stats
|
||||
total_schedulers = len(states)
|
||||
running = sum(1 for s in states if s.get('status') == 'running')
|
||||
stopped = total_schedulers - running
|
||||
total_executions = sum(s.get('executions', 0) for s in states)
|
||||
total_cost = sum(s.get('cost', 0.0) for s in states)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"📊 Aggregate Scheduler Statistics")
|
||||
print(f"{'='*60}")
|
||||
print(f"Total Schedulers: {total_schedulers}")
|
||||
print(f" 🟢 Running: {running}")
|
||||
print(f" 🔴 Stopped: {stopped}")
|
||||
print(f"\nTotal Executions: {total_executions}")
|
||||
print(f"Total Cost: ${total_cost:.4f}")
|
||||
|
||||
if total_executions > 0:
|
||||
avg_cost = total_cost / total_executions
|
||||
print(f"Avg Cost/Execution: ${avg_cost:.4f}")
|
||||
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Show per-scheduler breakdown
|
||||
print("Per-Scheduler Breakdown:")
|
||||
print(f"{'Name':<20} {'Status':<10} {'Executions':<12} {'Cost':<12}")
|
||||
print("="*60)
|
||||
|
||||
for state in sorted(states, key=lambda x: x.get('executions', 0), reverse=True):
|
||||
name = state['name'][:20]
|
||||
status = state.get('status', 'unknown')
|
||||
executions = state.get('executions', 0)
|
||||
cost = state.get('cost', 0.0)
|
||||
|
||||
status_icon = "🟢" if status == "running" else "🔴"
|
||||
print(f"{name:<20} {status_icon} {status:<8} {executions:<12} ${cost:<11.4f}")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_stop_all(state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule stop-all' command."""
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers running.")
|
||||
return 0
|
||||
|
||||
print(f"\n🛑 Stopping all schedulers ({len(states)} total)...\n")
|
||||
|
||||
stopped = 0
|
||||
failed = 0
|
||||
|
||||
for state in states:
|
||||
name = state['name']
|
||||
pid = state['pid']
|
||||
|
||||
try:
|
||||
if daemon_manager.stop_daemon(pid):
|
||||
state_manager.delete_state(name)
|
||||
print(f"✅ Stopped '{name}' (PID: {pid})")
|
||||
stopped += 1
|
||||
else:
|
||||
print(f"❌ Failed to stop '{name}' (PID: {pid})")
|
||||
failed += 1
|
||||
except Exception as e:
|
||||
print(f"❌ Error stopping '{name}': {e}")
|
||||
failed += 1
|
||||
|
||||
print(f"\n📊 Summary: {stopped} stopped, {failed} failed")
|
||||
return 0 if failed == 0 else 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_stop(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule stop' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule stop <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
print("\nList schedulers with: praisonai schedule list")
|
||||
return 1
|
||||
|
||||
pid = state.get('pid')
|
||||
if not pid:
|
||||
print(f"❌ Error: No PID found for scheduler '{name}'")
|
||||
return 1
|
||||
|
||||
print(f"🛑 Stopping scheduler '{name}' (PID: {pid})...")
|
||||
|
||||
success = daemon_manager.stop_daemon(pid)
|
||||
|
||||
if success:
|
||||
state['status'] = 'stopped'
|
||||
state_manager.save_state(name, state)
|
||||
print(f"✅ Scheduler '{name}' stopped successfully")
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Failed to stop scheduler '{name}'")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_logs(unknown_args, daemon_manager) -> int:
|
||||
"""Handle 'schedule logs' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule logs <name> [-f]")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
follow = '-f' in unknown_args or '--follow' in unknown_args
|
||||
|
||||
if follow:
|
||||
print(f"📋 Following logs for '{name}' (Ctrl+C to stop)...")
|
||||
import subprocess
|
||||
log_file = daemon_manager.log_dir / f"{name}.log"
|
||||
if not log_file.exists():
|
||||
print(f"❌ Error: Log file not found for '{name}'")
|
||||
return 1
|
||||
|
||||
try:
|
||||
subprocess.run(['tail', '-f', str(log_file)])
|
||||
except KeyboardInterrupt:
|
||||
print("\n✅ Stopped following logs")
|
||||
return 0
|
||||
else:
|
||||
logs = daemon_manager.read_logs(name, lines=50)
|
||||
if logs:
|
||||
print(f"📋 Last 50 lines of logs for '{name}':\n")
|
||||
print(logs)
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Error: No logs found for '{name}'")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_restart(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule restart' command."""
|
||||
from datetime import datetime
|
||||
import time
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule restart <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
print(f"🔄 Restarting scheduler '{name}'...")
|
||||
|
||||
# Stop if running
|
||||
pid = state.get('pid')
|
||||
if pid and state_manager.is_process_alive(pid):
|
||||
daemon_manager.stop_daemon(pid)
|
||||
time.sleep(1)
|
||||
|
||||
# Start again
|
||||
new_pid = daemon_manager.start_scheduler_daemon(
|
||||
name=name,
|
||||
task=state['task'],
|
||||
interval=state['interval'],
|
||||
max_cost=state.get('max_cost'),
|
||||
timeout=state.get('timeout'),
|
||||
max_retries=state.get('max_retries', 3)
|
||||
)
|
||||
|
||||
state['pid'] = new_pid
|
||||
state['status'] = 'running'
|
||||
state['started_at'] = datetime.now().isoformat()
|
||||
state_manager.save_state(name, state)
|
||||
|
||||
print(f"✅ Scheduler '{name}' restarted (PID: {new_pid})")
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_delete(unknown_args, state_manager) -> int:
|
||||
"""Handle 'schedule delete' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule delete <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
|
||||
if state_manager.delete_state(name):
|
||||
print(f"✅ Scheduler '{name}' deleted from list")
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_describe(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule describe' command."""
|
||||
from datetime import datetime
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule describe <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
print("\nList schedulers with: praisonai schedule list")
|
||||
return 1
|
||||
|
||||
# Get process status
|
||||
pid = state.get('pid', 0)
|
||||
is_alive = state_manager.is_process_alive(pid)
|
||||
status = "🟢 running" if is_alive else "🔴 stopped"
|
||||
|
||||
# Calculate uptime
|
||||
started_at = state.get('started_at')
|
||||
uptime = "N/A"
|
||||
if started_at:
|
||||
try:
|
||||
start_time = datetime.fromisoformat(started_at)
|
||||
uptime_delta = datetime.now() - start_time
|
||||
hours = int(uptime_delta.total_seconds() // 3600)
|
||||
minutes = int((uptime_delta.total_seconds() % 3600) // 60)
|
||||
uptime = f"{hours}h {minutes}m"
|
||||
except:
|
||||
pass
|
||||
|
||||
# Display detailed info
|
||||
print(f"\n{'='*60}")
|
||||
print(f"📋 Scheduler Details: {name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"Status: {status}")
|
||||
print(f"PID: {pid}")
|
||||
print(f"Uptime: {uptime}")
|
||||
print(f"Task: {state.get('task', 'N/A')}")
|
||||
print(f"Interval: {state.get('interval', 'N/A')}")
|
||||
print(f"Max Retries: {state.get('max_retries', 'N/A')}")
|
||||
|
||||
if state.get('timeout'):
|
||||
print(f"Timeout: {state['timeout']}s")
|
||||
|
||||
if state.get('max_cost'):
|
||||
print(f"Budget: ${state['max_cost']}")
|
||||
|
||||
print(f"Executions: {state.get('executions', 0)}")
|
||||
print(f"Total Cost: ${state.get('cost', 0.0):.4f}")
|
||||
print(f"Started: {started_at or 'N/A'}")
|
||||
|
||||
# Log file location
|
||||
log_file = daemon_manager.log_dir / f"{name}.log"
|
||||
print(f"Logs: {log_file}")
|
||||
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_save(unknown_args, state_manager) -> int:
|
||||
"""Handle 'schedule save' command."""
|
||||
import yaml
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule save <name> [output.yaml]")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
output_file = unknown_args[1] if len(unknown_args) > 1 else f"{name}.yaml"
|
||||
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
# Create YAML config from state
|
||||
yaml_config = {
|
||||
'framework': 'praisonai',
|
||||
'agents': [{
|
||||
'name': name,
|
||||
'role': 'Task Executor',
|
||||
'goal': state.get('task', ''),
|
||||
'instructions': state.get('task', ''),
|
||||
'verbose': True
|
||||
}],
|
||||
'task': state.get('task', ''),
|
||||
'schedule': {
|
||||
'interval': state.get('interval', 'hourly'),
|
||||
'max_retries': state.get('max_retries', 3),
|
||||
'run_immediately': True
|
||||
}
|
||||
}
|
||||
|
||||
# Add optional fields
|
||||
if state.get('timeout'):
|
||||
yaml_config['schedule']['timeout'] = state['timeout']
|
||||
|
||||
if state.get('max_cost'):
|
||||
yaml_config['schedule']['max_cost'] = state['max_cost']
|
||||
|
||||
# Write to file
|
||||
try:
|
||||
with open(output_file, 'w') as f:
|
||||
yaml.dump(yaml_config, f, default_flow_style=False, sort_keys=False)
|
||||
|
||||
print(f"✅ Configuration saved to: {output_file}")
|
||||
print(f"\nRun with:")
|
||||
print(f" praisonai schedule {output_file}")
|
||||
return 0
|
||||
except Exception as e:
|
||||
print(f"❌ Error saving configuration: {e}")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def handle_schedule_command(args, unknown_args=None) -> int:
|
||||
"""
|
||||
Handle the schedule command for running agents periodically.
|
||||
|
||||
Supports two modes:
|
||||
1. YAML mode: praisonai schedule agents.yaml
|
||||
2. Prompt mode: praisonai schedule "Your task here" --interval hourly
|
||||
|
||||
Args:
|
||||
args: Parsed command-line arguments
|
||||
unknown_args: Additional arguments after the command
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, 1 for error)
|
||||
"""
|
||||
try:
|
||||
from praisonai.scheduler import AgentScheduler
|
||||
from praisonaiagents import Agent
|
||||
except ImportError:
|
||||
print("Error: praisonai.scheduler or praisonaiagents module not found")
|
||||
print("Please ensure PraisonAI is properly installed")
|
||||
print("pip install praisonai praisonaiagents")
|
||||
return 1
|
||||
|
||||
# Check if first arg is a YAML file or a prompt
|
||||
first_arg = unknown_args[0] if unknown_args else None
|
||||
is_yaml_mode = first_arg and (first_arg.endswith('.yaml') or first_arg.endswith('.yml'))
|
||||
|
||||
# Get overrides from CLI
|
||||
interval_override = getattr(args, 'schedule_interval', None) or 'hourly'
|
||||
max_retries_override = getattr(args, 'schedule_max_retries', None) or 3
|
||||
timeout_override = getattr(args, 'timeout', None)
|
||||
max_cost_override = getattr(args, 'max_cost', None)
|
||||
verbose = getattr(args, 'verbose', False)
|
||||
|
||||
# Set up logging - only show logs if verbose
|
||||
if verbose:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='[%(asctime)s] %(levelname)s - %(message)s',
|
||||
datefmt='%Y-%m-%d %H:%M:%S'
|
||||
)
|
||||
else:
|
||||
# Suppress scheduler logs in non-verbose mode
|
||||
logging.getLogger('praisonai.scheduler').setLevel(logging.WARNING)
|
||||
|
||||
try:
|
||||
if is_yaml_mode:
|
||||
# YAML mode: Load from agents.yaml
|
||||
yaml_path = first_arg
|
||||
print(f"🤖 Loading agent configuration from: {yaml_path}")
|
||||
scheduler = AgentScheduler.from_yaml(
|
||||
yaml_path=yaml_path,
|
||||
interval_override=interval_override,
|
||||
max_retries_override=max_retries_override,
|
||||
timeout_override=timeout_override,
|
||||
max_cost_override=max_cost_override
|
||||
)
|
||||
else:
|
||||
# Prompt mode: Create agent from direct prompt
|
||||
if not first_arg:
|
||||
print("❌ Error: Please provide either a YAML file or a task prompt")
|
||||
print("\nExamples:")
|
||||
print(" praisonai schedule agents.yaml")
|
||||
print(' praisonai schedule "Check news every hour" --interval hourly')
|
||||
return 1
|
||||
|
||||
task = first_arg
|
||||
|
||||
# Create agent
|
||||
agent = Agent(
|
||||
name="Scheduled Agent",
|
||||
role="Task Executor",
|
||||
goal=task,
|
||||
instructions=task,
|
||||
verbose=False
|
||||
)
|
||||
|
||||
# Create scheduler
|
||||
scheduler = AgentScheduler(
|
||||
agent=agent,
|
||||
task=task,
|
||||
timeout=timeout_override,
|
||||
max_cost=max_cost_override
|
||||
)
|
||||
|
||||
# Get configuration
|
||||
interval = interval_override
|
||||
max_retries = max_retries_override
|
||||
|
||||
if is_yaml_mode:
|
||||
schedule_config = scheduler._yaml_schedule_config
|
||||
interval = interval_override or schedule_config.get('interval', 'hourly')
|
||||
max_retries = max_retries_override or schedule_config.get('max_retries', 3)
|
||||
|
||||
print(f"Schedule: {interval}")
|
||||
print(f"Max Retries: {max_retries}")
|
||||
if scheduler.timeout:
|
||||
print(f"Timeout: {scheduler.timeout}s")
|
||||
if scheduler.max_cost:
|
||||
print(f"Budget: ${scheduler.max_cost}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Start scheduler
|
||||
print("⏰ Starting scheduler... (Press Ctrl+C to stop)\n")
|
||||
if is_yaml_mode:
|
||||
scheduler.start_from_yaml_config()
|
||||
else:
|
||||
scheduler.start(
|
||||
schedule_expr=interval,
|
||||
max_retries=max_retries,
|
||||
run_immediately=True
|
||||
)
|
||||
|
||||
# Keep running until interrupted
|
||||
try:
|
||||
while scheduler.is_running:
|
||||
import time
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n🛑 Stopping scheduler...")
|
||||
scheduler.stop()
|
||||
|
||||
# Display final stats
|
||||
stats = scheduler.get_stats()
|
||||
print("\n📊 Final Statistics:")
|
||||
print(f" Total Executions: {stats['total_executions']}")
|
||||
print(f" Successful: {stats['successful_executions']}")
|
||||
print(f" Failed: {stats['failed_executions']}")
|
||||
print(f" Success Rate: {stats['success_rate']:.1f}%")
|
||||
print("\n✅ Agent stopped successfully\n")
|
||||
|
||||
return 0
|
||||
|
||||
except FileNotFoundError as e:
|
||||
print(f"❌ Error: {e}")
|
||||
print(f"\nMake sure {yaml_path} exists in the current directory.")
|
||||
print("\nExample agents.yaml structure:")
|
||||
print("""
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "AI News Monitor"
|
||||
role: "News Analyst"
|
||||
instructions: "Search and summarize AI news"
|
||||
tools:
|
||||
- search_tool
|
||||
|
||||
task: "Search for latest AI news"
|
||||
|
||||
schedule:
|
||||
interval: "hourly"
|
||||
max_retries: 3
|
||||
run_immediately: true
|
||||
""")
|
||||
return 1
|
||||
|
||||
except ValueError as e:
|
||||
print(f"❌ Configuration Error: {e}")
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {e}")
|
||||
if verbose:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def add_schedule_arguments(subparsers):
|
||||
"""
|
||||
Add schedule subcommand and arguments to parser.
|
||||
|
||||
Args:
|
||||
subparsers: Subparsers object from argparse
|
||||
"""
|
||||
schedule_parser = subparsers.add_parser(
|
||||
'schedule',
|
||||
help='Schedule an agent to run continuously at regular intervals'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'schedule_yaml',
|
||||
nargs='?',
|
||||
default='agents.yaml',
|
||||
help='Path to agents.yaml file (default: agents.yaml)'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--interval',
|
||||
dest='schedule_interval',
|
||||
type=str,
|
||||
help='Override schedule interval (e.g., "hourly", "*/30m", "daily")'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--max-retries',
|
||||
dest='schedule_max_retries',
|
||||
type=int,
|
||||
help='Override maximum retry attempts (default: from YAML or 3)'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--verbose', '-v',
|
||||
action='store_true',
|
||||
help='Enable verbose logging'
|
||||
)
|
||||
|
||||
schedule_parser.set_defaults(func=AgentSchedulerHandler.handle_schedule_command)
|
||||
|
||||
|
||||
def setup_scheduler_cli(subparsers):
|
||||
"""
|
||||
Setup scheduler CLI commands.
|
||||
|
||||
Args:
|
||||
subparsers: Subparsers object from argparse
|
||||
"""
|
||||
AgentSchedulerHandler.add_schedule_arguments(subparsers)
|
||||
@@ -0,0 +1,468 @@
|
||||
"""
|
||||
CLI Compare Feature for PraisonAI.
|
||||
|
||||
Compares different CLI command modes without performance impact.
|
||||
Only loaded when --compare flag is used.
|
||||
"""
|
||||
|
||||
import time
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, Any, List, Optional
|
||||
from datetime import datetime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
COMPARE_MODES = {
|
||||
"basic": {},
|
||||
"tools": {"tools": "internet_search"},
|
||||
"research": {"research": True},
|
||||
"planning": {"planning": True},
|
||||
"memory": {"memory": True},
|
||||
"router": {"router": True},
|
||||
"web_search": {"web_search": True},
|
||||
"web_fetch": {"web_fetch": True},
|
||||
"query_rewrite": {"query_rewrite": True},
|
||||
"expand_prompt": {"expand_prompt": True},
|
||||
}
|
||||
|
||||
|
||||
def get_mode_config(mode_name: str, custom_args: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Get configuration for a comparison mode.
|
||||
|
||||
Args:
|
||||
mode_name: Name of the mode
|
||||
custom_args: Optional custom arguments to merge
|
||||
|
||||
Returns:
|
||||
Configuration dictionary for the mode
|
||||
"""
|
||||
config = COMPARE_MODES.get(mode_name, {}).copy()
|
||||
if custom_args:
|
||||
config.update(custom_args)
|
||||
return config
|
||||
|
||||
|
||||
def list_available_modes() -> List[str]:
|
||||
"""
|
||||
List all available comparison modes.
|
||||
|
||||
Returns:
|
||||
List of mode names
|
||||
"""
|
||||
return list(COMPARE_MODES.keys())
|
||||
|
||||
|
||||
def parse_modes(modes_str: str) -> List[str]:
|
||||
"""
|
||||
Parse comma-separated modes string.
|
||||
|
||||
Args:
|
||||
modes_str: Comma-separated mode names
|
||||
|
||||
Returns:
|
||||
List of mode names
|
||||
"""
|
||||
if not modes_str or not modes_str.strip():
|
||||
return ["basic"]
|
||||
|
||||
modes = [m.strip() for m in modes_str.split(",")]
|
||||
return [m for m in modes if m]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModeResult:
|
||||
"""Result from running a single mode."""
|
||||
mode: str
|
||||
output: str
|
||||
execution_time_ms: float
|
||||
model_used: str
|
||||
tokens: Optional[Dict[str, int]] = None
|
||||
cost: Optional[float] = None
|
||||
tools_used: List[str] = field(default_factory=list)
|
||||
error: Optional[str] = None
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dictionary."""
|
||||
return {
|
||||
"mode": self.mode,
|
||||
"output_preview": self.output[:200] + "..." if len(self.output) > 200 else self.output,
|
||||
"execution_time_ms": self.execution_time_ms,
|
||||
"model_used": self.model_used,
|
||||
"tokens": self.tokens,
|
||||
"cost": self.cost,
|
||||
"tools_used": self.tools_used,
|
||||
"error": self.error,
|
||||
"timestamp": self.timestamp,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompareResult:
|
||||
"""Result from comparing multiple modes."""
|
||||
query: str
|
||||
comparisons: List[ModeResult] = field(default_factory=list)
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
|
||||
def get_summary(self) -> Dict[str, Any]:
|
||||
"""Generate summary statistics."""
|
||||
if not self.comparisons:
|
||||
return {}
|
||||
|
||||
times = [(c.mode, c.execution_time_ms) for c in self.comparisons if c.error is None]
|
||||
if not times:
|
||||
return {"error": "All modes failed"}
|
||||
|
||||
sorted_by_time = sorted(times, key=lambda x: x[1])
|
||||
|
||||
summary = {
|
||||
"fastest": sorted_by_time[0][0],
|
||||
"slowest": sorted_by_time[-1][0],
|
||||
"fastest_time_ms": sorted_by_time[0][1],
|
||||
"slowest_time_ms": sorted_by_time[-1][1],
|
||||
}
|
||||
|
||||
total_tokens = 0
|
||||
total_cost = 0.0
|
||||
for c in self.comparisons:
|
||||
if c.tokens:
|
||||
total_tokens += c.tokens.get("input", 0) + c.tokens.get("output", 0)
|
||||
if c.cost:
|
||||
total_cost += c.cost
|
||||
|
||||
if total_tokens > 0:
|
||||
summary["total_tokens"] = total_tokens
|
||||
if total_cost > 0:
|
||||
summary["total_cost"] = total_cost
|
||||
|
||||
return summary
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dictionary."""
|
||||
return {
|
||||
"query": self.query,
|
||||
"timestamp": self.timestamp,
|
||||
"comparisons": [c.to_dict() for c in self.comparisons],
|
||||
"summary": self.get_summary(),
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Convert to JSON string."""
|
||||
return json.dumps(self.to_dict(), indent=2)
|
||||
|
||||
|
||||
def save_compare_result(result: CompareResult, path: str) -> bool:
|
||||
"""
|
||||
Save comparison result to file.
|
||||
|
||||
Args:
|
||||
result: CompareResult to save
|
||||
path: File path
|
||||
|
||||
Returns:
|
||||
True if successful
|
||||
"""
|
||||
try:
|
||||
from pathlib import Path
|
||||
|
||||
file_path = Path(path)
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_path.write_text(result.to_json())
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to save comparison result: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def format_comparison_table(result: CompareResult) -> str:
|
||||
"""
|
||||
Format comparison result as a table string.
|
||||
|
||||
Args:
|
||||
result: CompareResult to format
|
||||
|
||||
Returns:
|
||||
Formatted table string
|
||||
"""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
from io import StringIO
|
||||
|
||||
console = Console(file=StringIO(), force_terminal=True)
|
||||
|
||||
table = Table(title=f"Comparison: {result.query[:50]}...")
|
||||
table.add_column("Mode", style="cyan")
|
||||
table.add_column("Time (ms)", style="green")
|
||||
table.add_column("Model", style="yellow")
|
||||
table.add_column("Tools", style="magenta")
|
||||
table.add_column("Status", style="bold")
|
||||
|
||||
for c in result.comparisons:
|
||||
status = "✅" if c.error is None else f"❌ {c.error[:20]}"
|
||||
tools = ", ".join(c.tools_used) if c.tools_used else "-"
|
||||
table.add_row(
|
||||
c.mode,
|
||||
f"{c.execution_time_ms:.1f}",
|
||||
c.model_used,
|
||||
tools,
|
||||
status
|
||||
)
|
||||
|
||||
summary = result.get_summary()
|
||||
if summary.get("fastest"):
|
||||
table.add_section()
|
||||
table.add_row(
|
||||
"Summary",
|
||||
f"Fastest: {summary['fastest']}",
|
||||
"",
|
||||
"",
|
||||
f"Δ {summary['slowest_time_ms'] - summary['fastest_time_ms']:.1f}ms"
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
return console.file.getvalue()
|
||||
except ImportError:
|
||||
lines = [f"Comparison: {result.query}"]
|
||||
lines.append("-" * 60)
|
||||
for c in result.comparisons:
|
||||
status = "OK" if c.error is None else f"ERROR: {c.error}"
|
||||
lines.append(f"{c.mode}: {c.execution_time_ms:.1f}ms - {status}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class CompareHandler:
|
||||
"""
|
||||
Handler for CLI compare feature.
|
||||
|
||||
Compares different CLI command modes and generates comparison reports.
|
||||
"""
|
||||
|
||||
def __init__(self, verbose: bool = False):
|
||||
"""
|
||||
Initialize the compare handler.
|
||||
|
||||
Args:
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
self.verbose = verbose
|
||||
self.results: List[ModeResult] = []
|
||||
|
||||
def compare(
|
||||
self,
|
||||
query: str,
|
||||
modes: List[str],
|
||||
model: Optional[str] = None,
|
||||
**kwargs
|
||||
) -> CompareResult:
|
||||
"""
|
||||
Compare query execution across different modes.
|
||||
|
||||
Args:
|
||||
query: The query to execute
|
||||
modes: List of mode names to compare
|
||||
model: Optional model override
|
||||
**kwargs: Additional arguments
|
||||
|
||||
Returns:
|
||||
CompareResult with all comparisons
|
||||
"""
|
||||
result = CompareResult(query=query)
|
||||
|
||||
for mode in modes:
|
||||
if self.verbose:
|
||||
logger.info(f"Running mode: {mode}")
|
||||
|
||||
try:
|
||||
mode_result = self._run_mode(query, mode, model=model, **kwargs)
|
||||
result.comparisons.append(mode_result)
|
||||
except Exception as e:
|
||||
logger.error(f"Error running mode {mode}: {e}")
|
||||
result.comparisons.append(ModeResult(
|
||||
mode=mode,
|
||||
output="",
|
||||
execution_time_ms=0,
|
||||
model_used=model or "unknown",
|
||||
error=str(e)
|
||||
))
|
||||
|
||||
return result
|
||||
|
||||
def _run_mode(
|
||||
self,
|
||||
query: str,
|
||||
mode: str,
|
||||
model: Optional[str] = None,
|
||||
**kwargs
|
||||
) -> ModeResult:
|
||||
"""
|
||||
Run a single mode and capture results.
|
||||
|
||||
Args:
|
||||
query: The query to execute
|
||||
mode: Mode name
|
||||
model: Optional model override
|
||||
**kwargs: Additional arguments
|
||||
|
||||
Returns:
|
||||
ModeResult with execution details
|
||||
"""
|
||||
config = get_mode_config(mode, kwargs)
|
||||
|
||||
if model:
|
||||
config["llm"] = model
|
||||
|
||||
start_time = time.perf_counter()
|
||||
output = ""
|
||||
tokens = None
|
||||
cost = None
|
||||
tools_used = []
|
||||
error = None
|
||||
model_used = model or "gpt-4o-mini"
|
||||
|
||||
try:
|
||||
from praisonaiagents import Agent
|
||||
|
||||
agent_config = {
|
||||
"name": f"CompareAgent_{mode}",
|
||||
"role": "Assistant",
|
||||
"goal": "Complete the given task",
|
||||
"backstory": "You are a helpful AI assistant",
|
||||
"verbose": False,
|
||||
}
|
||||
|
||||
if config.get("llm"):
|
||||
agent_config["llm"] = config["llm"]
|
||||
model_used = config["llm"]
|
||||
|
||||
if config.get("tools"):
|
||||
tools_list = self._load_tools(config["tools"])
|
||||
if tools_list:
|
||||
agent_config["tools"] = tools_list
|
||||
|
||||
if config.get("planning"):
|
||||
agent_config["planning"] = True
|
||||
|
||||
if config.get("memory"):
|
||||
agent_config["memory"] = True
|
||||
|
||||
if config.get("web_search"):
|
||||
agent_config["web_search"] = True
|
||||
|
||||
if config.get("web_fetch"):
|
||||
agent_config["web_fetch"] = True
|
||||
|
||||
agent = Agent(**agent_config)
|
||||
result = agent.start(query)
|
||||
|
||||
if hasattr(result, 'raw'):
|
||||
output = str(result.raw)
|
||||
else:
|
||||
output = str(result)
|
||||
|
||||
if hasattr(agent, 'tools') and agent.tools:
|
||||
for tool in agent.tools:
|
||||
if hasattr(tool, '__name__'):
|
||||
tools_used.append(tool.__name__)
|
||||
elif hasattr(tool, 'name'):
|
||||
tools_used.append(tool.name)
|
||||
|
||||
except Exception as e:
|
||||
error = str(e)
|
||||
logger.error(f"Mode {mode} failed: {e}")
|
||||
|
||||
end_time = time.perf_counter()
|
||||
execution_time_ms = (end_time - start_time) * 1000
|
||||
|
||||
return ModeResult(
|
||||
mode=mode,
|
||||
output=output,
|
||||
execution_time_ms=execution_time_ms,
|
||||
model_used=model_used,
|
||||
tokens=tokens,
|
||||
cost=cost,
|
||||
tools_used=tools_used,
|
||||
error=error
|
||||
)
|
||||
|
||||
def _load_tools(self, tools_str: str) -> List:
|
||||
"""
|
||||
Load tools from string specification.
|
||||
|
||||
Args:
|
||||
tools_str: Comma-separated tool names
|
||||
|
||||
Returns:
|
||||
List of tool functions
|
||||
"""
|
||||
tools = []
|
||||
tool_names = [t.strip() for t in tools_str.split(",")]
|
||||
|
||||
for name in tool_names:
|
||||
try:
|
||||
if name == "internet_search":
|
||||
from praisonaiagents.tools import internet_search
|
||||
tools.append(internet_search)
|
||||
elif name == "calculator":
|
||||
def calculator(expression: str) -> str:
|
||||
"""Evaluate a mathematical expression."""
|
||||
try:
|
||||
return str(eval(expression))
|
||||
except Exception as e:
|
||||
return f"Error: {e}"
|
||||
tools.append(calculator)
|
||||
except ImportError:
|
||||
logger.warning(f"Could not load tool: {name}")
|
||||
|
||||
return tools
|
||||
|
||||
def print_result(self, result: CompareResult) -> None:
|
||||
"""
|
||||
Print comparison result to console.
|
||||
|
||||
Args:
|
||||
result: CompareResult to print
|
||||
"""
|
||||
table_str = format_comparison_table(result)
|
||||
print(table_str)
|
||||
|
||||
def execute(
|
||||
self,
|
||||
query: str,
|
||||
modes_str: str,
|
||||
model: Optional[str] = None,
|
||||
output_path: Optional[str] = None,
|
||||
**kwargs
|
||||
) -> CompareResult:
|
||||
"""
|
||||
Execute comparison from CLI arguments.
|
||||
|
||||
Args:
|
||||
query: The query to execute
|
||||
modes_str: Comma-separated mode names
|
||||
model: Optional model override
|
||||
output_path: Optional path to save results
|
||||
**kwargs: Additional arguments
|
||||
|
||||
Returns:
|
||||
CompareResult with all comparisons
|
||||
"""
|
||||
modes = parse_modes(modes_str)
|
||||
|
||||
if self.verbose:
|
||||
print(f"[bold cyan]Comparing modes: {', '.join(modes)}[/bold cyan]")
|
||||
|
||||
result = self.compare(query, modes, model=model, **kwargs)
|
||||
|
||||
self.print_result(result)
|
||||
|
||||
if output_path:
|
||||
if save_compare_result(result, output_path):
|
||||
print(f"[green]Results saved to: {output_path}[/green]")
|
||||
else:
|
||||
print(f"[red]Failed to save results to: {output_path}[/red]")
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,471 @@
|
||||
"""
|
||||
Evaluation CLI feature for PraisonAI.
|
||||
|
||||
Provides CLI commands for running agent evaluations.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
from typing import Optional, List, Dict, Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EvalHandler:
|
||||
"""Handler for evaluation CLI commands."""
|
||||
|
||||
def __init__(self, verbose: bool = False):
|
||||
"""
|
||||
Initialize the evaluation handler.
|
||||
|
||||
Args:
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
self.verbose = verbose
|
||||
|
||||
def run_accuracy(
|
||||
self,
|
||||
agent_file: str,
|
||||
input_text: str,
|
||||
expected_output: str,
|
||||
iterations: int = 1,
|
||||
model: Optional[str] = None,
|
||||
output_file: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Run accuracy evaluation on an agent.
|
||||
|
||||
Args:
|
||||
agent_file: Path to agents.yaml file
|
||||
input_text: Input to provide to the agent
|
||||
expected_output: Expected output to compare against
|
||||
iterations: Number of evaluation iterations
|
||||
model: LLM model for judging
|
||||
output_file: Path to save results
|
||||
|
||||
Returns:
|
||||
Evaluation result dictionary
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents.eval import AccuracyEvaluator
|
||||
from praisonai.agents_generator import AgentsGenerator
|
||||
except ImportError as e:
|
||||
logger.error(f"Failed to import evaluation modules: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
try:
|
||||
generator = AgentsGenerator(agent_file)
|
||||
agents = generator.generate_agents()
|
||||
|
||||
if not agents:
|
||||
return {"error": "No agents found in configuration"}
|
||||
|
||||
agent = agents[0] if isinstance(agents, list) else agents
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text=input_text,
|
||||
expected_output=expected_output,
|
||||
num_iterations=iterations,
|
||||
model=model,
|
||||
save_results_path=output_file,
|
||||
verbose=self.verbose
|
||||
)
|
||||
|
||||
result = evaluator.run(print_summary=True)
|
||||
return result.to_dict()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Accuracy evaluation failed: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
def run_performance(
|
||||
self,
|
||||
agent_file: str,
|
||||
input_text: str = "Hello",
|
||||
iterations: int = 10,
|
||||
warmup: int = 2,
|
||||
track_memory: bool = True,
|
||||
output_file: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Run performance evaluation on an agent.
|
||||
|
||||
Args:
|
||||
agent_file: Path to agents.yaml file
|
||||
input_text: Input to provide to the agent
|
||||
iterations: Number of benchmark iterations
|
||||
warmup: Number of warmup runs
|
||||
track_memory: Whether to track memory usage
|
||||
output_file: Path to save results
|
||||
|
||||
Returns:
|
||||
Evaluation result dictionary
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents.eval import PerformanceEvaluator
|
||||
from praisonai.agents_generator import AgentsGenerator
|
||||
except ImportError as e:
|
||||
logger.error(f"Failed to import evaluation modules: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
try:
|
||||
generator = AgentsGenerator(agent_file)
|
||||
agents = generator.generate_agents()
|
||||
|
||||
if not agents:
|
||||
return {"error": "No agents found in configuration"}
|
||||
|
||||
agent = agents[0] if isinstance(agents, list) else agents
|
||||
|
||||
evaluator = PerformanceEvaluator(
|
||||
agent=agent,
|
||||
input_text=input_text,
|
||||
num_iterations=iterations,
|
||||
warmup_runs=warmup,
|
||||
track_memory=track_memory,
|
||||
save_results_path=output_file,
|
||||
verbose=self.verbose
|
||||
)
|
||||
|
||||
result = evaluator.run(print_summary=True)
|
||||
return result.to_dict()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Performance evaluation failed: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
def run_reliability(
|
||||
self,
|
||||
agent_file: str,
|
||||
input_text: str,
|
||||
expected_tools: List[str],
|
||||
forbidden_tools: Optional[List[str]] = None,
|
||||
output_file: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Run reliability evaluation on an agent.
|
||||
|
||||
Args:
|
||||
agent_file: Path to agents.yaml file
|
||||
input_text: Input to provide to the agent
|
||||
expected_tools: List of tools that should be called
|
||||
forbidden_tools: List of tools that should NOT be called
|
||||
output_file: Path to save results
|
||||
|
||||
Returns:
|
||||
Evaluation result dictionary
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents.eval import ReliabilityEvaluator
|
||||
from praisonai.agents_generator import AgentsGenerator
|
||||
except ImportError as e:
|
||||
logger.error(f"Failed to import evaluation modules: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
try:
|
||||
generator = AgentsGenerator(agent_file)
|
||||
agents = generator.generate_agents()
|
||||
|
||||
if not agents:
|
||||
return {"error": "No agents found in configuration"}
|
||||
|
||||
agent = agents[0] if isinstance(agents, list) else agents
|
||||
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text=input_text,
|
||||
expected_tools=expected_tools,
|
||||
forbidden_tools=forbidden_tools,
|
||||
save_results_path=output_file,
|
||||
verbose=self.verbose
|
||||
)
|
||||
|
||||
result = evaluator.run(print_summary=True)
|
||||
return result.to_dict()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Reliability evaluation failed: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
def run_criteria(
|
||||
self,
|
||||
agent_file: str,
|
||||
input_text: str,
|
||||
criteria: str,
|
||||
scoring_type: str = "numeric",
|
||||
threshold: float = 7.0,
|
||||
iterations: int = 1,
|
||||
model: Optional[str] = None,
|
||||
output_file: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Run criteria-based evaluation on an agent.
|
||||
|
||||
Args:
|
||||
agent_file: Path to agents.yaml file
|
||||
input_text: Input to provide to the agent
|
||||
criteria: Criteria to evaluate against
|
||||
scoring_type: "numeric" or "binary"
|
||||
threshold: Score threshold for passing (numeric mode)
|
||||
iterations: Number of evaluation iterations
|
||||
model: LLM model for judging
|
||||
output_file: Path to save results
|
||||
|
||||
Returns:
|
||||
Evaluation result dictionary
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents.eval import CriteriaEvaluator
|
||||
from praisonai.agents_generator import AgentsGenerator
|
||||
except ImportError as e:
|
||||
logger.error(f"Failed to import evaluation modules: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
try:
|
||||
generator = AgentsGenerator(agent_file)
|
||||
agents = generator.generate_agents()
|
||||
|
||||
if not agents:
|
||||
return {"error": "No agents found in configuration"}
|
||||
|
||||
agent = agents[0] if isinstance(agents, list) else agents
|
||||
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria=criteria,
|
||||
agent=agent,
|
||||
input_text=input_text,
|
||||
scoring_type=scoring_type,
|
||||
threshold=threshold,
|
||||
num_iterations=iterations,
|
||||
model=model,
|
||||
save_results_path=output_file,
|
||||
verbose=self.verbose
|
||||
)
|
||||
|
||||
result = evaluator.run(print_summary=True)
|
||||
return result.to_dict()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Criteria evaluation failed: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
def run_batch(
|
||||
self,
|
||||
agent_file: str,
|
||||
test_file: str,
|
||||
eval_type: str = "accuracy",
|
||||
output_file: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Run batch evaluation from a test file.
|
||||
|
||||
Args:
|
||||
agent_file: Path to agents.yaml file
|
||||
test_file: Path to JSON test file with test cases
|
||||
eval_type: Type of evaluation ("accuracy", "criteria")
|
||||
output_file: Path to save results
|
||||
|
||||
Returns:
|
||||
Batch evaluation results
|
||||
"""
|
||||
try:
|
||||
with open(test_file, 'r') as f:
|
||||
test_cases = json.load(f)
|
||||
except Exception as e:
|
||||
return {"error": f"Failed to load test file: {e}"}
|
||||
|
||||
results = []
|
||||
for i, test_case in enumerate(test_cases):
|
||||
if self.verbose:
|
||||
print(f"Running test case {i + 1}/{len(test_cases)}")
|
||||
|
||||
if eval_type == "accuracy":
|
||||
result = self.run_accuracy(
|
||||
agent_file=agent_file,
|
||||
input_text=test_case.get("input", ""),
|
||||
expected_output=test_case.get("expected", ""),
|
||||
iterations=test_case.get("iterations", 1)
|
||||
)
|
||||
elif eval_type == "criteria":
|
||||
result = self.run_criteria(
|
||||
agent_file=agent_file,
|
||||
input_text=test_case.get("input", ""),
|
||||
criteria=test_case.get("criteria", ""),
|
||||
scoring_type=test_case.get("scoring_type", "numeric"),
|
||||
threshold=test_case.get("threshold", 7.0)
|
||||
)
|
||||
else:
|
||||
result = {"error": f"Unknown eval type: {eval_type}"}
|
||||
|
||||
results.append({
|
||||
"test_case": i + 1,
|
||||
"input": test_case.get("input", ""),
|
||||
"result": result
|
||||
})
|
||||
|
||||
batch_result = {
|
||||
"total_tests": len(test_cases),
|
||||
"eval_type": eval_type,
|
||||
"results": results
|
||||
}
|
||||
|
||||
if output_file:
|
||||
try:
|
||||
with open(output_file, 'w') as f:
|
||||
json.dump(batch_result, f, indent=2)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save batch results: {e}")
|
||||
|
||||
return batch_result
|
||||
|
||||
|
||||
def handle_eval_command(args) -> None:
|
||||
"""
|
||||
Handle the eval CLI command.
|
||||
|
||||
Args:
|
||||
args: Parsed command line arguments
|
||||
"""
|
||||
handler = EvalHandler(verbose=getattr(args, 'verbose', False))
|
||||
|
||||
eval_type = getattr(args, 'eval_type', 'accuracy')
|
||||
agent_file = getattr(args, 'agent', 'agents.yaml')
|
||||
output_file = getattr(args, 'output', None)
|
||||
|
||||
if eval_type == 'accuracy':
|
||||
result = handler.run_accuracy(
|
||||
agent_file=agent_file,
|
||||
input_text=getattr(args, 'input', ''),
|
||||
expected_output=getattr(args, 'expected', ''),
|
||||
iterations=getattr(args, 'iterations', 1),
|
||||
model=getattr(args, 'model', None),
|
||||
output_file=output_file
|
||||
)
|
||||
elif eval_type == 'performance':
|
||||
result = handler.run_performance(
|
||||
agent_file=agent_file,
|
||||
input_text=getattr(args, 'input', 'Hello'),
|
||||
iterations=getattr(args, 'iterations', 10),
|
||||
warmup=getattr(args, 'warmup', 2),
|
||||
track_memory=getattr(args, 'memory', True),
|
||||
output_file=output_file
|
||||
)
|
||||
elif eval_type == 'reliability':
|
||||
expected_tools = getattr(args, 'expected_tools', '').split(',')
|
||||
forbidden_tools = getattr(args, 'forbidden_tools', '')
|
||||
forbidden_tools = forbidden_tools.split(',') if forbidden_tools else None
|
||||
|
||||
result = handler.run_reliability(
|
||||
agent_file=agent_file,
|
||||
input_text=getattr(args, 'input', ''),
|
||||
expected_tools=expected_tools,
|
||||
forbidden_tools=forbidden_tools,
|
||||
output_file=output_file
|
||||
)
|
||||
elif eval_type == 'criteria':
|
||||
result = handler.run_criteria(
|
||||
agent_file=agent_file,
|
||||
input_text=getattr(args, 'input', ''),
|
||||
criteria=getattr(args, 'criteria', ''),
|
||||
scoring_type=getattr(args, 'scoring', 'numeric'),
|
||||
threshold=getattr(args, 'threshold', 7.0),
|
||||
iterations=getattr(args, 'iterations', 1),
|
||||
model=getattr(args, 'model', None),
|
||||
output_file=output_file
|
||||
)
|
||||
elif eval_type == 'batch':
|
||||
result = handler.run_batch(
|
||||
agent_file=agent_file,
|
||||
test_file=getattr(args, 'test_file', ''),
|
||||
eval_type=getattr(args, 'batch_type', 'accuracy'),
|
||||
output_file=output_file
|
||||
)
|
||||
else:
|
||||
print(f"Unknown evaluation type: {eval_type}")
|
||||
return
|
||||
|
||||
if 'error' in result:
|
||||
print(f"Error: {result['error']}")
|
||||
elif not getattr(args, 'quiet', False):
|
||||
print(json.dumps(result, indent=2))
|
||||
|
||||
|
||||
def add_eval_parser(subparsers) -> None:
|
||||
"""
|
||||
Add eval subcommand parser.
|
||||
|
||||
Args:
|
||||
subparsers: Argument parser subparsers
|
||||
"""
|
||||
eval_parser = subparsers.add_parser(
|
||||
'eval',
|
||||
help='Run agent evaluations'
|
||||
)
|
||||
|
||||
eval_subparsers = eval_parser.add_subparsers(dest='eval_type')
|
||||
|
||||
accuracy_parser = eval_subparsers.add_parser(
|
||||
'accuracy',
|
||||
help='Run accuracy evaluation'
|
||||
)
|
||||
accuracy_parser.add_argument('--agent', '-a', default='agents.yaml', help='Agent config file')
|
||||
accuracy_parser.add_argument('--input', '-i', required=True, help='Input text')
|
||||
accuracy_parser.add_argument('--expected', '-e', required=True, help='Expected output')
|
||||
accuracy_parser.add_argument('--iterations', '-n', type=int, default=1, help='Number of iterations')
|
||||
accuracy_parser.add_argument('--model', '-m', help='Judge model')
|
||||
accuracy_parser.add_argument('--output', '-o', help='Output file')
|
||||
accuracy_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
|
||||
accuracy_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
|
||||
|
||||
perf_parser = eval_subparsers.add_parser(
|
||||
'performance',
|
||||
help='Run performance evaluation'
|
||||
)
|
||||
perf_parser.add_argument('--agent', '-a', default='agents.yaml', help='Agent config file')
|
||||
perf_parser.add_argument('--input', '-i', default='Hello', help='Input text')
|
||||
perf_parser.add_argument('--iterations', '-n', type=int, default=10, help='Number of iterations')
|
||||
perf_parser.add_argument('--warmup', '-w', type=int, default=2, help='Warmup runs')
|
||||
perf_parser.add_argument('--memory', action='store_true', default=True, help='Track memory')
|
||||
perf_parser.add_argument('--output', '-o', help='Output file')
|
||||
perf_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
|
||||
perf_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
|
||||
|
||||
rel_parser = eval_subparsers.add_parser(
|
||||
'reliability',
|
||||
help='Run reliability evaluation'
|
||||
)
|
||||
rel_parser.add_argument('--agent', '-a', default='agents.yaml', help='Agent config file')
|
||||
rel_parser.add_argument('--input', '-i', required=True, help='Input text')
|
||||
rel_parser.add_argument('--expected-tools', '-t', required=True, help='Expected tools (comma-separated)')
|
||||
rel_parser.add_argument('--forbidden-tools', '-f', help='Forbidden tools (comma-separated)')
|
||||
rel_parser.add_argument('--output', '-o', help='Output file')
|
||||
rel_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
|
||||
rel_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
|
||||
|
||||
criteria_parser = eval_subparsers.add_parser(
|
||||
'criteria',
|
||||
help='Run criteria-based evaluation'
|
||||
)
|
||||
criteria_parser.add_argument('--agent', '-a', default='agents.yaml', help='Agent config file')
|
||||
criteria_parser.add_argument('--input', '-i', required=True, help='Input text')
|
||||
criteria_parser.add_argument('--criteria', '-c', required=True, help='Evaluation criteria')
|
||||
criteria_parser.add_argument('--scoring', '-s', choices=['numeric', 'binary'], default='numeric', help='Scoring type')
|
||||
criteria_parser.add_argument('--threshold', type=float, default=7.0, help='Pass threshold')
|
||||
criteria_parser.add_argument('--iterations', '-n', type=int, default=1, help='Number of iterations')
|
||||
criteria_parser.add_argument('--model', '-m', help='Judge model')
|
||||
criteria_parser.add_argument('--output', '-o', help='Output file')
|
||||
criteria_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
|
||||
criteria_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
|
||||
|
||||
batch_parser = eval_subparsers.add_parser(
|
||||
'batch',
|
||||
help='Run batch evaluation from test file'
|
||||
)
|
||||
batch_parser.add_argument('--agent', '-a', default='agents.yaml', help='Agent config file')
|
||||
batch_parser.add_argument('--test-file', '-t', required=True, help='JSON test file')
|
||||
batch_parser.add_argument('--batch-type', '-b', choices=['accuracy', 'criteria'], default='accuracy', help='Evaluation type')
|
||||
batch_parser.add_argument('--output', '-o', help='Output file')
|
||||
batch_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
|
||||
batch_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
|
||||
@@ -0,0 +1,350 @@
|
||||
"""
|
||||
PraisonAI CLI Profiling Commands
|
||||
|
||||
Provides CLI commands for profiling agent execution.
|
||||
|
||||
Usage:
|
||||
praisonai profile run "task" --output report.html
|
||||
praisonai profile report --format json
|
||||
praisonai profile benchmark --iterations 10
|
||||
"""
|
||||
|
||||
import time
|
||||
import json
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
|
||||
class ProfilingHandler:
|
||||
"""
|
||||
Handler for CLI profiling commands.
|
||||
|
||||
Features:
|
||||
- Run tasks with profiling enabled
|
||||
- Generate reports (console, JSON, HTML)
|
||||
- Benchmark agent performance
|
||||
- Export flamegraphs
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._profiler = None
|
||||
|
||||
@property
|
||||
def profiler(self):
|
||||
"""Lazy load profiler."""
|
||||
if self._profiler is None:
|
||||
from praisonai.profiler import Profiler
|
||||
self._profiler = Profiler
|
||||
return self._profiler
|
||||
|
||||
def enable(self) -> None:
|
||||
"""Enable profiling."""
|
||||
self.profiler.enable()
|
||||
|
||||
def disable(self) -> None:
|
||||
"""Disable profiling."""
|
||||
self.profiler.disable()
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear profiling data."""
|
||||
self.profiler.clear()
|
||||
|
||||
def run_with_profiling(self, task: str, config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Run a task with profiling enabled.
|
||||
|
||||
Args:
|
||||
task: Task description or prompt
|
||||
config: Optional configuration dict
|
||||
|
||||
Returns:
|
||||
Dict with result and profiling data
|
||||
"""
|
||||
self.profiler.enable()
|
||||
self.profiler.clear()
|
||||
|
||||
result = None
|
||||
error = None
|
||||
|
||||
try:
|
||||
# Import and run PraisonAI
|
||||
from praisonai import PraisonAI
|
||||
|
||||
with self.profiler.block("total_execution"):
|
||||
with self.profiler.block("initialization"):
|
||||
praison = PraisonAI(auto=task, **(config or {}))
|
||||
|
||||
with self.profiler.block("agent_run"):
|
||||
result = praison.run()
|
||||
except Exception as e:
|
||||
error = str(e)
|
||||
finally:
|
||||
self.profiler.disable()
|
||||
|
||||
return {
|
||||
'result': result,
|
||||
'error': error,
|
||||
'summary': self.profiler.get_summary(),
|
||||
'statistics': self.profiler.get_statistics()
|
||||
}
|
||||
|
||||
def report(self, format: str = "console", output: Optional[str] = None) -> str:
|
||||
"""
|
||||
Generate profiling report.
|
||||
|
||||
Args:
|
||||
format: Output format (console, json, html)
|
||||
output: Optional file path to save report
|
||||
|
||||
Returns:
|
||||
Report content as string
|
||||
"""
|
||||
if format == "console":
|
||||
content = self.profiler.report(output="console")
|
||||
elif format == "json":
|
||||
content = self.profiler.export_json()
|
||||
elif format == "html":
|
||||
content = self.profiler.export_html()
|
||||
else:
|
||||
raise ValueError(f"Unknown format: {format}")
|
||||
|
||||
if output:
|
||||
with open(output, 'w') as f:
|
||||
f.write(content)
|
||||
print(f"Report saved to: {output}")
|
||||
|
||||
return content
|
||||
|
||||
def benchmark(self, task: str, iterations: int = 5,
|
||||
warmup: int = 1, config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Benchmark agent performance.
|
||||
|
||||
Args:
|
||||
task: Task to benchmark
|
||||
iterations: Number of iterations
|
||||
warmup: Number of warmup runs
|
||||
config: Optional configuration
|
||||
|
||||
Returns:
|
||||
Benchmark results with statistics
|
||||
"""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
times = []
|
||||
errors = []
|
||||
|
||||
# Warmup runs
|
||||
for i in range(warmup):
|
||||
print(f"Warmup {i + 1}/{warmup}...")
|
||||
try:
|
||||
self.run_with_profiling(task, config)
|
||||
except Exception as e:
|
||||
print(f"Warmup error: {e}")
|
||||
|
||||
# Benchmark runs
|
||||
for i in range(iterations):
|
||||
print(f"Iteration {i + 1}/{iterations}...")
|
||||
Profiler.clear()
|
||||
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
self.run_with_profiling(task, config)
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
times.append(duration_ms)
|
||||
except Exception as e:
|
||||
errors.append(str(e))
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
times.append(duration_ms)
|
||||
|
||||
# Calculate statistics
|
||||
if times:
|
||||
sorted_times = sorted(times)
|
||||
n = len(sorted_times)
|
||||
|
||||
results = {
|
||||
'iterations': iterations,
|
||||
'successful': iterations - len(errors),
|
||||
'failed': len(errors),
|
||||
'times_ms': times,
|
||||
'mean_ms': sum(times) / n,
|
||||
'min_ms': min(times),
|
||||
'max_ms': max(times),
|
||||
'p50_ms': sorted_times[n // 2],
|
||||
'p95_ms': sorted_times[int(n * 0.95)] if n > 1 else sorted_times[-1],
|
||||
'p99_ms': sorted_times[int(n * 0.99)] if n > 1 else sorted_times[-1],
|
||||
'errors': errors
|
||||
}
|
||||
else:
|
||||
results = {
|
||||
'iterations': iterations,
|
||||
'successful': 0,
|
||||
'failed': len(errors),
|
||||
'errors': errors
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
def export_flamegraph(self, filepath: str) -> None:
|
||||
"""Export flamegraph to SVG file."""
|
||||
self.profiler.export_flamegraph(filepath)
|
||||
print(f"Flamegraph saved to: {filepath}")
|
||||
|
||||
def get_summary(self) -> Dict[str, Any]:
|
||||
"""Get profiling summary."""
|
||||
return self.profiler.get_summary()
|
||||
|
||||
def get_statistics(self, category: Optional[str] = None) -> Dict[str, float]:
|
||||
"""Get profiling statistics."""
|
||||
return self.profiler.get_statistics(category)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print profiling summary to console."""
|
||||
summary = self.get_summary()
|
||||
stats = self.get_statistics()
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("PROFILING SUMMARY")
|
||||
print("=" * 60)
|
||||
|
||||
print(f"\nTotal Time: {summary['total_time_ms']:.2f}ms")
|
||||
print(f"Operations: {summary['timing_count']}")
|
||||
print(f"Imports: {summary['import_count']}")
|
||||
print(f"Flow Steps: {summary['flow_steps']}")
|
||||
|
||||
print("\nStatistics:")
|
||||
print(f" P50 (Median): {stats['p50']:.2f}ms")
|
||||
print(f" P95: {stats['p95']:.2f}ms")
|
||||
print(f" P99: {stats['p99']:.2f}ms")
|
||||
print(f" Mean: {stats['mean']:.2f}ms")
|
||||
print(f" Std Dev: {stats['std_dev']:.2f}ms")
|
||||
|
||||
print("\nSlowest Operations:")
|
||||
for name, duration in summary['slowest_operations'][:5]:
|
||||
print(f" {name}: {duration:.2f}ms")
|
||||
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
def handle_profile_command(args) -> int:
|
||||
"""
|
||||
Handle profile CLI command.
|
||||
|
||||
Args:
|
||||
args: Parsed command line arguments
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success)
|
||||
"""
|
||||
handler = ProfilingHandler()
|
||||
|
||||
subcommand = getattr(args, 'profile_command', None)
|
||||
|
||||
if subcommand == 'run':
|
||||
task = getattr(args, 'task', '')
|
||||
output = getattr(args, 'output', None)
|
||||
format_type = getattr(args, 'format', 'console')
|
||||
|
||||
print(f"Running with profiling: {task[:50]}...")
|
||||
result = handler.run_with_profiling(task)
|
||||
|
||||
if result['error']:
|
||||
print(f"Error: {result['error']}")
|
||||
|
||||
handler.report(format=format_type, output=output)
|
||||
return 0 if not result['error'] else 1
|
||||
|
||||
elif subcommand == 'report':
|
||||
output = getattr(args, 'output', None)
|
||||
format_type = getattr(args, 'format', 'console')
|
||||
|
||||
handler.report(format=format_type, output=output)
|
||||
return 0
|
||||
|
||||
elif subcommand == 'benchmark':
|
||||
task = getattr(args, 'task', '')
|
||||
iterations = getattr(args, 'iterations', 5)
|
||||
warmup = getattr(args, 'warmup', 1)
|
||||
output = getattr(args, 'output', None)
|
||||
|
||||
print(f"Benchmarking: {task[:50]}...")
|
||||
results = handler.benchmark(task, iterations=iterations, warmup=warmup)
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("BENCHMARK RESULTS")
|
||||
print("=" * 60)
|
||||
print(f"Iterations: {results['iterations']}")
|
||||
print(f"Successful: {results['successful']}")
|
||||
print(f"Failed: {results['failed']}")
|
||||
|
||||
if 'mean_ms' in results:
|
||||
print("\nTiming:")
|
||||
print(f" Mean: {results['mean_ms']:.2f}ms")
|
||||
print(f" Min: {results['min_ms']:.2f}ms")
|
||||
print(f" Max: {results['max_ms']:.2f}ms")
|
||||
print(f" P50: {results['p50_ms']:.2f}ms")
|
||||
print(f" P95: {results['p95_ms']:.2f}ms")
|
||||
|
||||
if output:
|
||||
with open(output, 'w') as f:
|
||||
json.dump(results, f, indent=2)
|
||||
print(f"\nResults saved to: {output}")
|
||||
|
||||
return 0
|
||||
|
||||
elif subcommand == 'flamegraph':
|
||||
output = getattr(args, 'output', 'profile.svg')
|
||||
handler.export_flamegraph(output)
|
||||
return 0
|
||||
|
||||
elif subcommand == 'summary':
|
||||
handler.print_summary()
|
||||
return 0
|
||||
|
||||
else:
|
||||
print("Usage: praisonai profile <run|report|benchmark|flamegraph|summary>")
|
||||
return 1
|
||||
|
||||
|
||||
def add_profile_parser(subparsers) -> None:
|
||||
"""Add profile subcommand to argument parser."""
|
||||
profile_parser = subparsers.add_parser(
|
||||
'profile',
|
||||
help='Profile agent execution'
|
||||
)
|
||||
|
||||
profile_subparsers = profile_parser.add_subparsers(dest='profile_command')
|
||||
|
||||
# Run with profiling
|
||||
run_parser = profile_subparsers.add_parser('run', help='Run task with profiling')
|
||||
run_parser.add_argument('task', help='Task to run')
|
||||
run_parser.add_argument('--output', '-o', help='Output file path')
|
||||
run_parser.add_argument('--format', '-f', choices=['console', 'json', 'html'],
|
||||
default='console', help='Output format')
|
||||
|
||||
# Generate report
|
||||
report_parser = profile_subparsers.add_parser('report', help='Generate profiling report')
|
||||
report_parser.add_argument('--output', '-o', help='Output file path')
|
||||
report_parser.add_argument('--format', '-f', choices=['console', 'json', 'html'],
|
||||
default='console', help='Output format')
|
||||
|
||||
# Benchmark
|
||||
bench_parser = profile_subparsers.add_parser('benchmark', help='Benchmark agent performance')
|
||||
bench_parser.add_argument('task', help='Task to benchmark')
|
||||
bench_parser.add_argument('--iterations', '-n', type=int, default=5, help='Number of iterations')
|
||||
bench_parser.add_argument('--warmup', '-w', type=int, default=1, help='Warmup runs')
|
||||
bench_parser.add_argument('--output', '-o', help='Output file for results')
|
||||
|
||||
# Flamegraph
|
||||
flame_parser = profile_subparsers.add_parser('flamegraph', help='Export flamegraph')
|
||||
flame_parser.add_argument('--output', '-o', default='profile.svg', help='Output SVG file')
|
||||
|
||||
# Summary
|
||||
profile_subparsers.add_parser('summary', help='Print profiling summary') # noqa: F841
|
||||
|
||||
|
||||
# Exports
|
||||
__all__ = [
|
||||
'ProfilingHandler',
|
||||
'handle_profile_command',
|
||||
'add_profile_parser',
|
||||
]
|
||||
@@ -0,0 +1,310 @@
|
||||
"""
|
||||
Skills CLI Feature Handler
|
||||
|
||||
Provides CLI commands for managing Agent Skills:
|
||||
- list: List available skills
|
||||
- validate: Validate a skill directory
|
||||
- create: Create a new skill from template
|
||||
- prompt: Generate prompt XML for skills
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional, List
|
||||
|
||||
|
||||
class SkillsHandler:
|
||||
"""Handler for skills CLI commands."""
|
||||
|
||||
def __init__(self, verbose: bool = True):
|
||||
"""Initialize the skills handler.
|
||||
|
||||
Args:
|
||||
verbose: Whether to print verbose output
|
||||
"""
|
||||
self.verbose = verbose
|
||||
|
||||
def list_skills(
|
||||
self,
|
||||
skill_dirs: Optional[List[str]] = None,
|
||||
include_defaults: bool = True
|
||||
) -> List[dict]:
|
||||
"""List all available skills.
|
||||
|
||||
Args:
|
||||
skill_dirs: Optional list of directories to scan
|
||||
include_defaults: Whether to include default skill directories
|
||||
|
||||
Returns:
|
||||
List of skill info dictionaries
|
||||
"""
|
||||
from praisonaiagents.skills import discover_skills
|
||||
|
||||
skills = discover_skills(skill_dirs, include_defaults)
|
||||
|
||||
result = []
|
||||
for skill in skills:
|
||||
info = {
|
||||
"name": skill.name,
|
||||
"description": skill.description,
|
||||
"path": str(skill.path) if skill.path else None,
|
||||
"license": skill.license,
|
||||
}
|
||||
result.append(info)
|
||||
|
||||
if self.verbose:
|
||||
print(f" {skill.name}: {skill.description[:60]}...")
|
||||
|
||||
if self.verbose and not result:
|
||||
print("No skills found.")
|
||||
|
||||
return result
|
||||
|
||||
def validate_skill(self, skill_path: str) -> dict:
|
||||
"""Validate a skill directory.
|
||||
|
||||
Args:
|
||||
skill_path: Path to the skill directory
|
||||
|
||||
Returns:
|
||||
Validation result dictionary with 'valid' and 'errors' keys
|
||||
"""
|
||||
from praisonaiagents.skills import validate
|
||||
|
||||
path = Path(skill_path).expanduser().resolve()
|
||||
errors = validate(path)
|
||||
|
||||
result = {
|
||||
"valid": len(errors) == 0,
|
||||
"path": str(path),
|
||||
"errors": errors
|
||||
}
|
||||
|
||||
if self.verbose:
|
||||
if result["valid"]:
|
||||
print(f"✓ Skill at {path} is valid")
|
||||
else:
|
||||
print(f"✗ Skill at {path} has errors:")
|
||||
for error in errors:
|
||||
print(f" - {error}")
|
||||
|
||||
return result
|
||||
|
||||
def create_skill(
|
||||
self,
|
||||
name: str,
|
||||
description: str = "A custom skill",
|
||||
output_dir: Optional[str] = None
|
||||
) -> str:
|
||||
"""Create a new skill from template.
|
||||
|
||||
Args:
|
||||
name: Skill name (kebab-case)
|
||||
description: Skill description
|
||||
output_dir: Directory to create skill in (default: current dir)
|
||||
|
||||
Returns:
|
||||
Path to created skill directory
|
||||
"""
|
||||
# Validate name format
|
||||
import re
|
||||
if not re.match(r'^[a-z][a-z0-9-]*[a-z0-9]$', name) and len(name) > 1:
|
||||
if not re.match(r'^[a-z]$', name):
|
||||
raise ValueError(
|
||||
f"Invalid skill name '{name}'. "
|
||||
"Must be lowercase, use hyphens, and not start/end with hyphen."
|
||||
)
|
||||
|
||||
base_dir = Path(output_dir or os.getcwd())
|
||||
skill_dir = base_dir / name
|
||||
|
||||
if skill_dir.exists():
|
||||
raise ValueError(f"Directory already exists: {skill_dir}")
|
||||
|
||||
# Create directory structure
|
||||
skill_dir.mkdir(parents=True)
|
||||
(skill_dir / "scripts").mkdir()
|
||||
(skill_dir / "references").mkdir()
|
||||
(skill_dir / "assets").mkdir()
|
||||
|
||||
# Create SKILL.md
|
||||
skill_md_content = f"""---
|
||||
name: {name}
|
||||
description: {description}
|
||||
license: Apache-2.0
|
||||
compatibility: Works with PraisonAI Agents
|
||||
metadata:
|
||||
author: user
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
# {name.replace('-', ' ').title()}
|
||||
|
||||
## Overview
|
||||
|
||||
{description}
|
||||
|
||||
## Usage
|
||||
|
||||
Describe how to use this skill.
|
||||
|
||||
## Instructions
|
||||
|
||||
1. Step one
|
||||
2. Step two
|
||||
3. Step three
|
||||
"""
|
||||
(skill_dir / "SKILL.md").write_text(skill_md_content)
|
||||
|
||||
# Create placeholder files
|
||||
(skill_dir / "scripts" / ".gitkeep").write_text("")
|
||||
(skill_dir / "references" / ".gitkeep").write_text("")
|
||||
(skill_dir / "assets" / ".gitkeep").write_text("")
|
||||
|
||||
if self.verbose:
|
||||
print(f"✓ Created skill at {skill_dir}")
|
||||
print(" - SKILL.md")
|
||||
print(" - scripts/")
|
||||
print(" - references/")
|
||||
print(" - assets/")
|
||||
|
||||
return str(skill_dir)
|
||||
|
||||
def generate_prompt(
|
||||
self,
|
||||
skill_dirs: Optional[List[str]] = None,
|
||||
include_defaults: bool = True
|
||||
) -> str:
|
||||
"""Generate prompt XML for available skills.
|
||||
|
||||
Args:
|
||||
skill_dirs: Optional list of directories to scan
|
||||
include_defaults: Whether to include default skill directories
|
||||
|
||||
Returns:
|
||||
XML string with <available_skills> block
|
||||
"""
|
||||
from praisonaiagents.skills import SkillManager
|
||||
|
||||
manager = SkillManager()
|
||||
|
||||
if skill_dirs:
|
||||
manager.discover(skill_dirs, include_defaults=include_defaults)
|
||||
elif include_defaults:
|
||||
manager.discover(include_defaults=True)
|
||||
|
||||
prompt = manager.to_prompt()
|
||||
|
||||
if self.verbose:
|
||||
print(prompt)
|
||||
|
||||
return prompt
|
||||
|
||||
|
||||
def handle_skills_command(args) -> int:
|
||||
"""Handle skills subcommand from CLI.
|
||||
|
||||
Args:
|
||||
args: Parsed command line arguments
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, non-zero for error)
|
||||
"""
|
||||
handler = SkillsHandler(verbose=True)
|
||||
|
||||
try:
|
||||
if args.skills_command == "list":
|
||||
dirs = args.dirs if hasattr(args, 'dirs') and args.dirs else None
|
||||
handler.list_skills(dirs, include_defaults=True)
|
||||
|
||||
elif args.skills_command == "validate":
|
||||
if not hasattr(args, 'path') or not args.path:
|
||||
print("Error: --path is required for validate command")
|
||||
return 1
|
||||
result = handler.validate_skill(args.path)
|
||||
return 0 if result["valid"] else 1
|
||||
|
||||
elif args.skills_command == "create":
|
||||
if not hasattr(args, 'name') or not args.name:
|
||||
print("Error: --name is required for create command")
|
||||
return 1
|
||||
description = args.description if hasattr(args, 'description') else "A custom skill"
|
||||
output_dir = args.output if hasattr(args, 'output') else None
|
||||
handler.create_skill(args.name, description, output_dir)
|
||||
|
||||
elif args.skills_command == "prompt":
|
||||
dirs = args.dirs if hasattr(args, 'dirs') and args.dirs else None
|
||||
handler.generate_prompt(dirs, include_defaults=True)
|
||||
|
||||
else:
|
||||
print(f"Unknown skills command: {args.skills_command}")
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def add_skills_parser(subparsers) -> None:
|
||||
"""Add skills subcommand to argument parser.
|
||||
|
||||
Args:
|
||||
subparsers: Subparsers object from argparse
|
||||
"""
|
||||
# subparsers is already the skills subparsers object
|
||||
# We just need to add the subcommands to it
|
||||
|
||||
# list command
|
||||
list_parser = subparsers.add_parser(
|
||||
'list',
|
||||
help='List available skills'
|
||||
)
|
||||
list_parser.add_argument(
|
||||
'--dirs',
|
||||
nargs='+',
|
||||
help='Directories to scan for skills'
|
||||
)
|
||||
|
||||
# validate command
|
||||
validate_parser = subparsers.add_parser(
|
||||
'validate',
|
||||
help='Validate a skill directory'
|
||||
)
|
||||
validate_parser.add_argument(
|
||||
'--path',
|
||||
required=True,
|
||||
help='Path to skill directory'
|
||||
)
|
||||
|
||||
# create command
|
||||
create_parser = subparsers.add_parser(
|
||||
'create',
|
||||
help='Create a new skill from template'
|
||||
)
|
||||
create_parser.add_argument(
|
||||
'--name',
|
||||
required=True,
|
||||
help='Skill name (kebab-case)'
|
||||
)
|
||||
create_parser.add_argument(
|
||||
'--description',
|
||||
default='A custom skill',
|
||||
help='Skill description'
|
||||
)
|
||||
create_parser.add_argument(
|
||||
'--output',
|
||||
help='Output directory (default: current directory)'
|
||||
)
|
||||
|
||||
# prompt command
|
||||
prompt_parser = subparsers.add_parser(
|
||||
'prompt',
|
||||
help='Generate prompt XML for skills'
|
||||
)
|
||||
prompt_parser.add_argument(
|
||||
'--dirs',
|
||||
nargs='+',
|
||||
help='Directories to scan for skills'
|
||||
)
|
||||
@@ -236,17 +236,30 @@ class PraisonAI:
|
||||
# Check for subcommands (start, list, stop, logs, restart)
|
||||
subcommand = unknown_args[0] if unknown_args and not unknown_args[0].startswith('-') else None
|
||||
|
||||
if subcommand in ['start', 'list', 'stop', 'logs', 'restart', 'delete']:
|
||||
if subcommand in ['start', 'list', 'stop', 'logs', 'restart', 'delete', 'describe', 'save', 'stop-all', 'stats']:
|
||||
exit_code = AgentSchedulerHandler.handle_daemon_command(subcommand, args, unknown_args[1:] if len(unknown_args) > 1 else [])
|
||||
else:
|
||||
# Legacy mode: direct scheduling (foreground)
|
||||
exit_code = AgentSchedulerHandler.handle_schedule_command(args, unknown_args)
|
||||
# Legacy mode: direct scheduling (foreground) or daemon mode
|
||||
daemon_mode = getattr(args, 'daemon', False)
|
||||
exit_code = AgentSchedulerHandler.handle_schedule_command(args, unknown_args, daemon_mode=daemon_mode)
|
||||
|
||||
sys.exit(exit_code)
|
||||
elif args.command.startswith("tests.test") or args.command.startswith("tests/test"): # Argument used for testing purposes
|
||||
print("test")
|
||||
return "test"
|
||||
else:
|
||||
# Handle --compare flag for CLI mode comparison
|
||||
if hasattr(args, 'compare') and args.compare:
|
||||
from .features.compare import CompareHandler
|
||||
handler = CompareHandler(verbose=getattr(args, 'verbose', False))
|
||||
result = handler.execute(
|
||||
args.command,
|
||||
args.compare,
|
||||
model=getattr(args, 'llm', None),
|
||||
output_path=getattr(args, 'compare_output', None)
|
||||
)
|
||||
return result
|
||||
|
||||
# Combine command with any available inputs (stdin and/or file)
|
||||
combined_inputs = []
|
||||
if stdin_input:
|
||||
@@ -600,7 +613,7 @@ class PraisonAI:
|
||||
return default_args
|
||||
|
||||
# Define special commands
|
||||
special_commands = ['chat', 'code', 'call', 'realtime', 'train', 'ui', 'context', 'research', 'memory', 'rules', 'workflow', 'hooks', 'knowledge', 'session', 'tools', 'todo', 'docs', 'mcp', 'commit', 'serve', 'schedule']
|
||||
special_commands = ['chat', 'code', 'call', 'realtime', 'train', 'ui', 'context', 'research', 'memory', 'rules', 'workflow', 'hooks', 'knowledge', 'session', 'tools', 'todo', 'docs', 'mcp', 'commit', 'serve', 'schedule', 'skills']
|
||||
|
||||
parser = argparse.ArgumentParser(prog="praisonai", description="praisonAI command-line interface")
|
||||
parser.add_argument("--framework", choices=["crewai", "autogen", "praisonai"], help="Specify the framework")
|
||||
@@ -730,11 +743,16 @@ class PraisonAI:
|
||||
parser.add_argument("--external-agent", type=str, choices=["claude", "gemini", "codex", "cursor"],
|
||||
help="Use external AI CLI tool (claude, gemini, codex, cursor)")
|
||||
|
||||
# Compare - compare different CLI modes
|
||||
parser.add_argument("--compare", type=str, help="Compare CLI modes (comma-separated: basic,tools,research,planning)")
|
||||
parser.add_argument("--compare-output", type=str, help="Save comparison results to file")
|
||||
|
||||
# Agent Scheduler - for schedule command
|
||||
parser.add_argument("--interval", dest="schedule_interval", type=str, help="Schedule interval (e.g., 'hourly', '*/30m', 'daily')")
|
||||
parser.add_argument("--schedule-max-retries", dest="schedule_max_retries", type=int, help="Maximum retry attempts for scheduled execution")
|
||||
parser.add_argument("--timeout", type=int, help="Maximum execution time per run in seconds")
|
||||
parser.add_argument("--max-cost", dest="max_cost", type=float, help="Maximum total cost budget in USD")
|
||||
parser.add_argument('--max-cost', type=float, help='Maximum total cost budget in USD')
|
||||
parser.add_argument('--daemon', action='store_true', help=argparse.SUPPRESS) # Hidden flag for daemon mode
|
||||
|
||||
# If we're in a test environment, parse with empty args to avoid pytest interference
|
||||
if in_test_env:
|
||||
@@ -796,7 +814,7 @@ class PraisonAI:
|
||||
except ModuleNotFoundError as e:
|
||||
missing_module = str(e).split("'")[1]
|
||||
print(f"[red]ERROR: Missing dependency {missing_module}. Install with:[/red]")
|
||||
print(f"\npip install \"praisonai[chat]\"\n")
|
||||
print("\npip install \"praisonai[chat]\"\n")
|
||||
sys.exit(1)
|
||||
sys.exit(0)
|
||||
|
||||
@@ -1024,6 +1042,23 @@ class PraisonAI:
|
||||
self.handle_commit_command(unknown_args)
|
||||
sys.exit(0)
|
||||
|
||||
elif args.command == 'skills':
|
||||
if not PRAISONAI_AVAILABLE:
|
||||
print("[red]ERROR: PraisonAI Agents is not installed. Install with:[/red]")
|
||||
print("\npip install praisonaiagents\n")
|
||||
sys.exit(1)
|
||||
|
||||
from .features.skills import handle_skills_command, add_skills_parser
|
||||
|
||||
# Create a parser for skills command
|
||||
skills_parser = argparse.ArgumentParser(prog="praisonai skills")
|
||||
skills_subparsers = skills_parser.add_subparsers(dest='skills_command', help='Skills commands')
|
||||
add_skills_parser(skills_subparsers)
|
||||
skills_args = skills_parser.parse_args(unknown_args)
|
||||
|
||||
exit_code = handle_skills_command(skills_args)
|
||||
sys.exit(exit_code)
|
||||
|
||||
# Only check framework availability for agent-related operations
|
||||
if not args.command and (args.init or args.auto or args.framework):
|
||||
if not CREWAI_AVAILABLE and not AUTOGEN_AVAILABLE and not PRAISONAI_AVAILABLE:
|
||||
|
||||
@@ -57,7 +57,7 @@ class CloudDeployer:
|
||||
file.write("FROM python:3.11-slim\n")
|
||||
file.write("WORKDIR /app\n")
|
||||
file.write("COPY . .\n")
|
||||
file.write("RUN pip install flask praisonai==2.3.67 gunicorn markdown\n")
|
||||
file.write("RUN pip install flask praisonai==2.3.75 gunicorn markdown\n")
|
||||
file.write("EXPOSE 8080\n")
|
||||
file.write('CMD ["gunicorn", "-b", "0.0.0.0:8080", "api:app"]\n')
|
||||
|
||||
|
||||
@@ -8,7 +8,14 @@ Features:
|
||||
- Function execution timing
|
||||
- Flow tracking
|
||||
- File/module usage tracking
|
||||
- Memory usage (optional)
|
||||
- Memory usage (tracemalloc)
|
||||
- API call profiling (wall-clock time)
|
||||
- Streaming profiling (TTFT, total time)
|
||||
- Statistics (p50, p95, p99)
|
||||
- cProfile integration
|
||||
- Flamegraph generation
|
||||
- Line-level profiling
|
||||
- JSON/HTML export
|
||||
|
||||
Usage:
|
||||
from praisonai.profiler import Profiler, profile, profile_imports
|
||||
@@ -22,12 +29,27 @@ Usage:
|
||||
with Profiler.block("my_operation"):
|
||||
do_something()
|
||||
|
||||
# Profile API calls
|
||||
with Profiler.api_call("https://api.example.com") as call:
|
||||
response = requests.get(...)
|
||||
|
||||
# Profile streaming
|
||||
with Profiler.streaming("chat") as tracker:
|
||||
tracker.first_token()
|
||||
for chunk in stream:
|
||||
tracker.chunk()
|
||||
|
||||
# Profile imports
|
||||
with profile_imports():
|
||||
import heavy_module
|
||||
|
||||
# Get report
|
||||
# Get report with statistics
|
||||
Profiler.report()
|
||||
stats = Profiler.get_statistics()
|
||||
|
||||
# Export
|
||||
Profiler.export_json()
|
||||
Profiler.export_html()
|
||||
"""
|
||||
|
||||
import time
|
||||
@@ -35,9 +57,15 @@ import functools
|
||||
import threading
|
||||
import sys
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
import json
|
||||
import tracemalloc
|
||||
import cProfile
|
||||
import pstats
|
||||
import io
|
||||
import statistics
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from typing import Dict, List, Optional, Callable, Any
|
||||
from contextlib import contextmanager
|
||||
from contextlib import contextmanager, asynccontextmanager
|
||||
|
||||
|
||||
# ============================================================================
|
||||
@@ -55,6 +83,38 @@ class TimingRecord:
|
||||
timestamp: float = field(default_factory=time.time)
|
||||
|
||||
|
||||
@dataclass
|
||||
class APICallRecord:
|
||||
"""Record of an API/HTTP call."""
|
||||
endpoint: str
|
||||
method: str
|
||||
duration_ms: float
|
||||
status_code: int = 0
|
||||
request_size: int = 0
|
||||
response_size: int = 0
|
||||
timestamp: float = field(default_factory=time.time)
|
||||
|
||||
|
||||
@dataclass
|
||||
class StreamingRecord:
|
||||
"""Record of streaming operation (LLM responses)."""
|
||||
name: str
|
||||
ttft_ms: float # Time to first token
|
||||
total_ms: float
|
||||
chunk_count: int = 0
|
||||
total_tokens: int = 0
|
||||
timestamp: float = field(default_factory=time.time)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MemoryRecord:
|
||||
"""Record of memory usage."""
|
||||
name: str
|
||||
current_kb: float
|
||||
peak_kb: float
|
||||
timestamp: float = field(default_factory=time.time)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportRecord:
|
||||
"""Record of a module import."""
|
||||
@@ -75,6 +135,80 @@ class FlowRecord:
|
||||
timestamp: float = field(default_factory=time.time)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Streaming Tracker
|
||||
# ============================================================================
|
||||
|
||||
class StreamingTracker:
|
||||
"""
|
||||
Track streaming operations (LLM responses).
|
||||
|
||||
Usage:
|
||||
tracker = StreamingTracker("chat")
|
||||
tracker.start()
|
||||
tracker.first_token() # Mark TTFT
|
||||
for chunk in stream:
|
||||
tracker.chunk()
|
||||
tracker.end(total_tokens=100)
|
||||
"""
|
||||
|
||||
def __init__(self, name: str):
|
||||
self.name = name
|
||||
self._start_time: Optional[float] = None
|
||||
self._first_token_time: Optional[float] = None
|
||||
self._end_time: Optional[float] = None
|
||||
self._chunk_count: int = 0
|
||||
self._total_tokens: int = 0
|
||||
|
||||
def start(self) -> None:
|
||||
"""Start tracking."""
|
||||
self._start_time = time.perf_counter()
|
||||
|
||||
def first_token(self) -> None:
|
||||
"""Mark time to first token."""
|
||||
if self._first_token_time is None:
|
||||
self._first_token_time = time.perf_counter()
|
||||
|
||||
def chunk(self) -> None:
|
||||
"""Record a chunk received."""
|
||||
self._chunk_count += 1
|
||||
|
||||
def end(self, total_tokens: int = 0) -> None:
|
||||
"""End tracking and record to Profiler."""
|
||||
self._end_time = time.perf_counter()
|
||||
self._total_tokens = total_tokens
|
||||
|
||||
if self._start_time is not None:
|
||||
ttft_ms = 0.0
|
||||
if self._first_token_time is not None:
|
||||
ttft_ms = (self._first_token_time - self._start_time) * 1000
|
||||
|
||||
total_ms = (self._end_time - self._start_time) * 1000
|
||||
|
||||
Profiler.record_streaming(
|
||||
name=self.name,
|
||||
ttft_ms=ttft_ms,
|
||||
total_ms=total_ms,
|
||||
chunk_count=self._chunk_count,
|
||||
total_tokens=self._total_tokens
|
||||
)
|
||||
|
||||
@property
|
||||
def ttft_ms(self) -> float:
|
||||
"""Get time to first token in ms."""
|
||||
if self._start_time and self._first_token_time:
|
||||
return (self._first_token_time - self._start_time) * 1000
|
||||
return 0.0
|
||||
|
||||
@property
|
||||
def elapsed_ms(self) -> float:
|
||||
"""Get elapsed time in ms."""
|
||||
if self._start_time:
|
||||
end = self._end_time or time.perf_counter()
|
||||
return (end - self._start_time) * 1000
|
||||
return 0.0
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Profiler Class
|
||||
# ============================================================================
|
||||
@@ -84,6 +218,16 @@ class Profiler:
|
||||
Centralized profiler for performance monitoring.
|
||||
|
||||
Thread-safe singleton pattern for global access.
|
||||
|
||||
Features:
|
||||
- Function/block timing
|
||||
- API call profiling (wall-clock)
|
||||
- Streaming profiling (TTFT)
|
||||
- Memory profiling
|
||||
- Import timing
|
||||
- Statistics (p50, p95, p99)
|
||||
- cProfile integration
|
||||
- Export (JSON, HTML)
|
||||
"""
|
||||
|
||||
_instance: Optional['Profiler'] = None
|
||||
@@ -93,9 +237,14 @@ class Profiler:
|
||||
_timings: List[TimingRecord] = []
|
||||
_imports: List[ImportRecord] = []
|
||||
_flow: List[FlowRecord] = []
|
||||
_api_calls: List[APICallRecord] = []
|
||||
_streaming: List[StreamingRecord] = []
|
||||
_memory: List[MemoryRecord] = []
|
||||
_enabled: bool = False
|
||||
_flow_step: int = 0
|
||||
_files_accessed: Dict[str, int] = {}
|
||||
_line_profile_data: Dict[str, Any] = {}
|
||||
_cprofile_stats: List[Dict[str, Any]] = []
|
||||
|
||||
def __new__(cls):
|
||||
if cls._instance is None:
|
||||
@@ -126,8 +275,13 @@ class Profiler:
|
||||
cls._timings.clear()
|
||||
cls._imports.clear()
|
||||
cls._flow.clear()
|
||||
cls._api_calls.clear()
|
||||
cls._streaming.clear()
|
||||
cls._memory.clear()
|
||||
cls._flow_step = 0
|
||||
cls._files_accessed.clear()
|
||||
cls._line_profile_data.clear()
|
||||
cls._cprofile_stats.clear()
|
||||
|
||||
@classmethod
|
||||
def record_timing(cls, name: str, duration_ms: float, category: str = "function",
|
||||
@@ -295,6 +449,433 @@ class Profiler:
|
||||
print(report_text)
|
||||
|
||||
return report_text
|
||||
|
||||
# ========================================================================
|
||||
# API Call Profiling
|
||||
# ========================================================================
|
||||
|
||||
@classmethod
|
||||
def record_api_call(cls, endpoint: str, method: str, duration_ms: float,
|
||||
status_code: int = 0, request_size: int = 0,
|
||||
response_size: int = 0) -> None:
|
||||
"""Record an API/HTTP call timing."""
|
||||
if not cls.is_enabled():
|
||||
return
|
||||
|
||||
with cls._lock:
|
||||
cls._api_calls.append(APICallRecord(
|
||||
endpoint=endpoint,
|
||||
method=method,
|
||||
duration_ms=duration_ms,
|
||||
status_code=status_code,
|
||||
request_size=request_size,
|
||||
response_size=response_size
|
||||
))
|
||||
|
||||
@classmethod
|
||||
def get_api_calls(cls) -> List[APICallRecord]:
|
||||
"""Get API call records."""
|
||||
with cls._lock:
|
||||
return cls._api_calls.copy()
|
||||
|
||||
@classmethod
|
||||
@contextmanager
|
||||
def api_call(cls, endpoint: str, method: str = "GET"):
|
||||
"""Context manager for profiling API calls."""
|
||||
start = time.perf_counter()
|
||||
call_info = {'status_code': 0, 'request_size': 0, 'response_size': 0}
|
||||
|
||||
try:
|
||||
yield call_info
|
||||
finally:
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
cls.record_api_call(
|
||||
endpoint=endpoint,
|
||||
method=method,
|
||||
duration_ms=duration_ms,
|
||||
status_code=call_info.get('status_code', 0),
|
||||
request_size=call_info.get('request_size', 0),
|
||||
response_size=call_info.get('response_size', 0)
|
||||
)
|
||||
|
||||
# ========================================================================
|
||||
# Streaming Profiling
|
||||
# ========================================================================
|
||||
|
||||
@classmethod
|
||||
def record_streaming(cls, name: str, ttft_ms: float, total_ms: float,
|
||||
chunk_count: int = 0, total_tokens: int = 0) -> None:
|
||||
"""Record streaming metrics."""
|
||||
if not cls.is_enabled():
|
||||
return
|
||||
|
||||
with cls._lock:
|
||||
cls._streaming.append(StreamingRecord(
|
||||
name=name,
|
||||
ttft_ms=ttft_ms,
|
||||
total_ms=total_ms,
|
||||
chunk_count=chunk_count,
|
||||
total_tokens=total_tokens
|
||||
))
|
||||
|
||||
@classmethod
|
||||
def get_streaming_records(cls) -> List[StreamingRecord]:
|
||||
"""Get streaming records."""
|
||||
with cls._lock:
|
||||
return cls._streaming.copy()
|
||||
|
||||
@classmethod
|
||||
@contextmanager
|
||||
def streaming(cls, name: str):
|
||||
"""Context manager for profiling streaming operations."""
|
||||
tracker = StreamingTracker(name)
|
||||
tracker.start()
|
||||
try:
|
||||
yield tracker
|
||||
finally:
|
||||
tracker.end()
|
||||
|
||||
@classmethod
|
||||
@asynccontextmanager
|
||||
async def streaming_async(cls, name: str):
|
||||
"""Async context manager for profiling streaming operations."""
|
||||
tracker = StreamingTracker(name)
|
||||
tracker.start()
|
||||
try:
|
||||
yield tracker
|
||||
finally:
|
||||
tracker.end()
|
||||
|
||||
# ========================================================================
|
||||
# Memory Profiling
|
||||
# ========================================================================
|
||||
|
||||
@classmethod
|
||||
def record_memory(cls, name: str, current_kb: float, peak_kb: float) -> None:
|
||||
"""Record memory usage."""
|
||||
if not cls.is_enabled():
|
||||
return
|
||||
|
||||
with cls._lock:
|
||||
cls._memory.append(MemoryRecord(
|
||||
name=name,
|
||||
current_kb=current_kb,
|
||||
peak_kb=peak_kb
|
||||
))
|
||||
|
||||
@classmethod
|
||||
def get_memory_records(cls) -> List[MemoryRecord]:
|
||||
"""Get memory records."""
|
||||
with cls._lock:
|
||||
return cls._memory.copy()
|
||||
|
||||
@classmethod
|
||||
@contextmanager
|
||||
def memory(cls, name: str):
|
||||
"""Context manager for profiling memory usage."""
|
||||
tracemalloc.start()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
current, peak = tracemalloc.get_traced_memory()
|
||||
tracemalloc.stop()
|
||||
cls.record_memory(name, current / 1024, peak / 1024)
|
||||
|
||||
@classmethod
|
||||
def memory_snapshot(cls) -> Dict[str, float]:
|
||||
"""Take a memory snapshot."""
|
||||
tracemalloc.start()
|
||||
current, peak = tracemalloc.get_traced_memory()
|
||||
tracemalloc.stop()
|
||||
return {
|
||||
'current_kb': current / 1024,
|
||||
'peak_kb': peak / 1024
|
||||
}
|
||||
|
||||
# ========================================================================
|
||||
# Statistics
|
||||
# ========================================================================
|
||||
|
||||
@classmethod
|
||||
def get_statistics(cls, category: Optional[str] = None) -> Dict[str, float]:
|
||||
"""
|
||||
Get statistical analysis of timing data.
|
||||
|
||||
Returns p50, p95, p99, mean, std_dev, min, max.
|
||||
"""
|
||||
with cls._lock:
|
||||
if category:
|
||||
durations = [t.duration_ms for t in cls._timings if t.category == category]
|
||||
else:
|
||||
durations = [t.duration_ms for t in cls._timings]
|
||||
|
||||
if not durations:
|
||||
return {
|
||||
'p50': 0.0, 'p95': 0.0, 'p99': 0.0,
|
||||
'mean': 0.0, 'std_dev': 0.0, 'min': 0.0, 'max': 0.0,
|
||||
'count': 0
|
||||
}
|
||||
|
||||
sorted_durations = sorted(durations)
|
||||
n = len(sorted_durations)
|
||||
|
||||
def percentile(p: float) -> float:
|
||||
idx = int(n * p / 100)
|
||||
return sorted_durations[min(idx, n - 1)]
|
||||
|
||||
mean = statistics.mean(durations)
|
||||
std_dev = statistics.stdev(durations) if n > 1 else 0.0
|
||||
|
||||
return {
|
||||
'p50': percentile(50),
|
||||
'p95': percentile(95),
|
||||
'p99': percentile(99),
|
||||
'mean': mean,
|
||||
'std_dev': std_dev,
|
||||
'min': min(durations),
|
||||
'max': max(durations),
|
||||
'count': n
|
||||
}
|
||||
|
||||
# ========================================================================
|
||||
# cProfile Integration
|
||||
# ========================================================================
|
||||
|
||||
@classmethod
|
||||
@contextmanager
|
||||
def cprofile(cls, name: str):
|
||||
"""Context manager for cProfile profiling."""
|
||||
pr = cProfile.Profile()
|
||||
pr.enable()
|
||||
|
||||
try:
|
||||
yield pr
|
||||
finally:
|
||||
pr.disable()
|
||||
|
||||
# Store stats
|
||||
s = io.StringIO()
|
||||
ps = pstats.Stats(pr, stream=s).sort_stats('cumulative')
|
||||
ps.print_stats(30)
|
||||
|
||||
with cls._lock:
|
||||
cls._cprofile_stats.append({
|
||||
'name': name,
|
||||
'stats': s.getvalue(),
|
||||
'total_calls': ps.total_calls if hasattr(ps, 'total_calls') else 0,
|
||||
'timestamp': time.time()
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def get_cprofile_stats(cls) -> List[Dict[str, Any]]:
|
||||
"""Get cProfile statistics."""
|
||||
with cls._lock:
|
||||
return cls._cprofile_stats.copy()
|
||||
|
||||
# ========================================================================
|
||||
# Line-Level Profiling
|
||||
# ========================================================================
|
||||
|
||||
@classmethod
|
||||
def get_line_profile_data(cls) -> Dict[str, Any]:
|
||||
"""Get line-level profiling data."""
|
||||
with cls._lock:
|
||||
return cls._line_profile_data.copy()
|
||||
|
||||
@classmethod
|
||||
def set_line_profile_data(cls, func_name: str, data: Any) -> None:
|
||||
"""Store line-level profiling data."""
|
||||
if not cls.is_enabled():
|
||||
return
|
||||
with cls._lock:
|
||||
cls._line_profile_data[func_name] = data
|
||||
|
||||
# ========================================================================
|
||||
# Flamegraph
|
||||
# ========================================================================
|
||||
|
||||
@classmethod
|
||||
def get_flamegraph_data(cls) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Generate flamegraph-compatible data from flow records.
|
||||
|
||||
Returns list of {name, value, children} for flamegraph visualization.
|
||||
"""
|
||||
with cls._lock:
|
||||
# Convert flow records to flamegraph format
|
||||
data = []
|
||||
for record in cls._flow:
|
||||
data.append({
|
||||
'name': record.name,
|
||||
'value': record.duration_ms,
|
||||
'file': record.file,
|
||||
'line': record.line
|
||||
})
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
def export_flamegraph(cls, filepath: str) -> None:
|
||||
"""
|
||||
Export flamegraph to SVG file.
|
||||
|
||||
Note: Requires flamegraph data. For full flamegraph support,
|
||||
use py-spy: py-spy record -o profile.svg -- python script.py
|
||||
"""
|
||||
data = cls.get_flamegraph_data()
|
||||
|
||||
# Generate simple SVG flamegraph
|
||||
svg_content = cls._generate_simple_flamegraph_svg(data)
|
||||
|
||||
with open(filepath, 'w') as f:
|
||||
f.write(svg_content)
|
||||
|
||||
@classmethod
|
||||
def _generate_simple_flamegraph_svg(cls, data: List[Dict[str, Any]]) -> str:
|
||||
"""Generate a simple SVG flamegraph."""
|
||||
if not data:
|
||||
return '<svg xmlns="http://www.w3.org/2000/svg" width="800" height="100"><text x="10" y="50">No profiling data</text></svg>'
|
||||
|
||||
total_time = sum(d['value'] for d in data)
|
||||
if total_time == 0:
|
||||
total_time = 1
|
||||
|
||||
width = 800
|
||||
height = max(100, len(data) * 25 + 50)
|
||||
|
||||
svg_parts = [
|
||||
f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}">',
|
||||
'<style>rect:hover { opacity: 0.8; } text { font-family: monospace; font-size: 12px; }</style>',
|
||||
f'<text x="10" y="20">PraisonAI Profiling Flamegraph (Total: {total_time:.2f}ms)</text>'
|
||||
]
|
||||
|
||||
y = 40
|
||||
for item in sorted(data, key=lambda x: x['value'], reverse=True)[:20]:
|
||||
bar_width = max(10, (item['value'] / total_time) * (width - 20))
|
||||
color = f'hsl({int(item["value"] / total_time * 120)}, 70%, 50%)'
|
||||
|
||||
svg_parts.append(
|
||||
f'<rect x="10" y="{y}" width="{bar_width}" height="20" fill="{color}" />'
|
||||
)
|
||||
svg_parts.append(
|
||||
f'<text x="15" y="{y + 15}">{item["name"]}: {item["value"]:.2f}ms</text>'
|
||||
)
|
||||
y += 25
|
||||
|
||||
svg_parts.append('</svg>')
|
||||
return '\n'.join(svg_parts)
|
||||
|
||||
# ========================================================================
|
||||
# Export Functions
|
||||
# ========================================================================
|
||||
|
||||
@classmethod
|
||||
def export_json(cls) -> str:
|
||||
"""Export profiling data as JSON."""
|
||||
# Get summary and stats first (they acquire their own locks)
|
||||
summary = cls.get_summary()
|
||||
stats = cls.get_statistics()
|
||||
|
||||
with cls._lock:
|
||||
data = {
|
||||
'summary': summary,
|
||||
'statistics': stats,
|
||||
'timings': [asdict(t) for t in cls._timings],
|
||||
'api_calls': [asdict(a) for a in cls._api_calls],
|
||||
'streaming': [asdict(s) for s in cls._streaming],
|
||||
'memory': [asdict(m) for m in cls._memory],
|
||||
'imports': [asdict(i) for i in cls._imports],
|
||||
'flow': [asdict(f) for f in cls._flow]
|
||||
}
|
||||
return json.dumps(data, indent=2, default=str)
|
||||
|
||||
@classmethod
|
||||
def export_html(cls) -> str:
|
||||
"""Export profiling data as HTML report."""
|
||||
summary = cls.get_summary()
|
||||
stats = cls.get_statistics()
|
||||
|
||||
html = f'''<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>PraisonAI Profiling Report</title>
|
||||
<style>
|
||||
body {{ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; margin: 20px; }}
|
||||
h1 {{ color: #333; }}
|
||||
h2 {{ color: #666; border-bottom: 1px solid #ddd; padding-bottom: 5px; }}
|
||||
table {{ border-collapse: collapse; width: 100%; margin: 10px 0; }}
|
||||
th, td {{ border: 1px solid #ddd; padding: 8px; text-align: left; }}
|
||||
th {{ background-color: #f5f5f5; }}
|
||||
.metric {{ display: inline-block; margin: 10px; padding: 15px; background: #f9f9f9; border-radius: 5px; }}
|
||||
.metric-value {{ font-size: 24px; font-weight: bold; color: #0066cc; }}
|
||||
.metric-label {{ font-size: 12px; color: #666; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>PraisonAI Profiling Report</h1>
|
||||
|
||||
<h2>Summary</h2>
|
||||
<div class="metric">
|
||||
<div class="metric-value">{summary['total_time_ms']:.2f}ms</div>
|
||||
<div class="metric-label">Total Time</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="metric-value">{summary['timing_count']}</div>
|
||||
<div class="metric-label">Operations</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="metric-value">{len(cls._api_calls)}</div>
|
||||
<div class="metric-label">API Calls</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="metric-value">{len(cls._streaming)}</div>
|
||||
<div class="metric-label">Streams</div>
|
||||
</div>
|
||||
|
||||
<h2>Statistics</h2>
|
||||
<table>
|
||||
<tr><th>Metric</th><th>Value</th></tr>
|
||||
<tr><td>P50 (Median)</td><td>{stats['p50']:.2f}ms</td></tr>
|
||||
<tr><td>P95</td><td>{stats['p95']:.2f}ms</td></tr>
|
||||
<tr><td>P99</td><td>{stats['p99']:.2f}ms</td></tr>
|
||||
<tr><td>Mean</td><td>{stats['mean']:.2f}ms</td></tr>
|
||||
<tr><td>Std Dev</td><td>{stats['std_dev']:.2f}ms</td></tr>
|
||||
<tr><td>Min</td><td>{stats['min']:.2f}ms</td></tr>
|
||||
<tr><td>Max</td><td>{stats['max']:.2f}ms</td></tr>
|
||||
</table>
|
||||
|
||||
<h2>Slowest Operations</h2>
|
||||
<table>
|
||||
<tr><th>Operation</th><th>Duration (ms)</th></tr>
|
||||
{''.join(f"<tr><td>{name}</td><td>{dur:.2f}</td></tr>" for name, dur in summary['slowest_operations'])}
|
||||
</table>
|
||||
|
||||
<h2>API Calls</h2>
|
||||
<table>
|
||||
<tr><th>Endpoint</th><th>Method</th><th>Duration (ms)</th><th>Status</th></tr>
|
||||
{''.join(f"<tr><td>{a.endpoint}</td><td>{a.method}</td><td>{a.duration_ms:.2f}</td><td>{a.status_code}</td></tr>" for a in cls._api_calls[:20])}
|
||||
</table>
|
||||
|
||||
<h2>Streaming</h2>
|
||||
<table>
|
||||
<tr><th>Name</th><th>TTFT (ms)</th><th>Total (ms)</th><th>Chunks</th><th>Tokens</th></tr>
|
||||
{''.join(f"<tr><td>{s.name}</td><td>{s.ttft_ms:.2f}</td><td>{s.total_ms:.2f}</td><td>{s.chunk_count}</td><td>{s.total_tokens}</td></tr>" for s in cls._streaming[:20])}
|
||||
</table>
|
||||
</body>
|
||||
</html>'''
|
||||
return html
|
||||
|
||||
@classmethod
|
||||
def export_to_file(cls, filepath: str, format: str = "json") -> None:
|
||||
"""Export profiling data to file."""
|
||||
if format == "json":
|
||||
content = cls.export_json()
|
||||
elif format == "html":
|
||||
content = cls.export_html()
|
||||
else:
|
||||
raise ValueError(f"Unknown format: {format}")
|
||||
|
||||
with open(filepath, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
@@ -422,6 +1003,161 @@ def profile_imports():
|
||||
return ImportProfiler()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# API Profiling Decorators
|
||||
# ============================================================================
|
||||
|
||||
def profile_api(func: Optional[Callable] = None, *, endpoint: str = ""):
|
||||
"""
|
||||
Decorator to profile a function as an API call.
|
||||
|
||||
Usage:
|
||||
@profile_api(endpoint="openai/chat")
|
||||
def call_openai():
|
||||
pass
|
||||
"""
|
||||
def decorator(fn: Callable) -> Callable:
|
||||
@functools.wraps(fn)
|
||||
def wrapper(*args, **kwargs):
|
||||
if not Profiler.is_enabled():
|
||||
return fn(*args, **kwargs)
|
||||
|
||||
ep = endpoint or fn.__name__
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
finally:
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
Profiler.record_api_call(ep, "CALL", duration_ms)
|
||||
|
||||
return wrapper
|
||||
|
||||
if func is not None:
|
||||
return decorator(func)
|
||||
return decorator
|
||||
|
||||
|
||||
def profile_api_async(func: Optional[Callable] = None, *, endpoint: str = ""):
|
||||
"""
|
||||
Decorator to profile an async function as an API call.
|
||||
"""
|
||||
def decorator(fn: Callable) -> Callable:
|
||||
@functools.wraps(fn)
|
||||
async def wrapper(*args, **kwargs):
|
||||
if not Profiler.is_enabled():
|
||||
return await fn(*args, **kwargs)
|
||||
|
||||
ep = endpoint or fn.__name__
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
return await fn(*args, **kwargs)
|
||||
finally:
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
Profiler.record_api_call(ep, "CALL", duration_ms)
|
||||
|
||||
return wrapper
|
||||
|
||||
if func is not None:
|
||||
return decorator(func)
|
||||
return decorator
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# cProfile Decorator
|
||||
# ============================================================================
|
||||
|
||||
def profile_detailed(func: Optional[Callable] = None):
|
||||
"""
|
||||
Decorator for detailed cProfile profiling.
|
||||
|
||||
Usage:
|
||||
@profile_detailed
|
||||
def heavy_computation():
|
||||
pass
|
||||
"""
|
||||
def decorator(fn: Callable) -> Callable:
|
||||
@functools.wraps(fn)
|
||||
def wrapper(*args, **kwargs):
|
||||
if not Profiler.is_enabled():
|
||||
return fn(*args, **kwargs)
|
||||
|
||||
pr = cProfile.Profile()
|
||||
pr.enable()
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
finally:
|
||||
pr.disable()
|
||||
s = io.StringIO()
|
||||
ps = pstats.Stats(pr, stream=s).sort_stats('cumulative')
|
||||
ps.print_stats(20)
|
||||
Profiler._cprofile_stats.append({
|
||||
'name': fn.__name__,
|
||||
'stats': s.getvalue(),
|
||||
'timestamp': time.time()
|
||||
})
|
||||
|
||||
return wrapper
|
||||
|
||||
if func is not None:
|
||||
return decorator(func)
|
||||
return decorator
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Line-Level Profiling Decorator
|
||||
# ============================================================================
|
||||
|
||||
def profile_lines(func: Optional[Callable] = None):
|
||||
"""
|
||||
Decorator for line-level profiling.
|
||||
|
||||
Note: Requires line_profiler package for full functionality.
|
||||
Falls back to basic timing if not available.
|
||||
|
||||
Usage:
|
||||
@profile_lines
|
||||
def my_function():
|
||||
pass
|
||||
"""
|
||||
def decorator(fn: Callable) -> Callable:
|
||||
@functools.wraps(fn)
|
||||
def wrapper(*args, **kwargs):
|
||||
if not Profiler.is_enabled():
|
||||
return fn(*args, **kwargs)
|
||||
|
||||
# Try to use line_profiler if available
|
||||
try:
|
||||
from line_profiler import LineProfiler
|
||||
lp = LineProfiler()
|
||||
lp.add_function(fn)
|
||||
lp.enable()
|
||||
try:
|
||||
result = fn(*args, **kwargs)
|
||||
finally:
|
||||
lp.disable()
|
||||
s = io.StringIO()
|
||||
lp.print_stats(stream=s)
|
||||
Profiler.set_line_profile_data(fn.__name__, s.getvalue())
|
||||
return result
|
||||
except ImportError:
|
||||
# Fallback to basic timing
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
finally:
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
Profiler.set_line_profile_data(fn.__name__, {
|
||||
'note': 'line_profiler not installed',
|
||||
'total_ms': duration_ms
|
||||
})
|
||||
|
||||
return wrapper
|
||||
|
||||
if func is not None:
|
||||
return decorator(func)
|
||||
return decorator
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Quick Profiling Functions
|
||||
# ============================================================================
|
||||
@@ -452,14 +1188,27 @@ def check_module_available(module_name: str) -> bool:
|
||||
# ============================================================================
|
||||
|
||||
__all__ = [
|
||||
# Core
|
||||
'Profiler',
|
||||
'StreamingTracker',
|
||||
# Decorators
|
||||
'profile',
|
||||
'profile_async',
|
||||
'profile_api',
|
||||
'profile_api_async',
|
||||
'profile_detailed',
|
||||
'profile_lines',
|
||||
# Import profiling
|
||||
'profile_imports',
|
||||
'ImportProfiler',
|
||||
# Utilities
|
||||
'time_import',
|
||||
'check_module_available',
|
||||
# Data classes
|
||||
'TimingRecord',
|
||||
'ImportRecord',
|
||||
'FlowRecord',
|
||||
'APICallRecord',
|
||||
'StreamingRecord',
|
||||
'MemoryRecord',
|
||||
]
|
||||
@@ -100,16 +100,16 @@ class AgentScheduler:
|
||||
self.is_running = True
|
||||
self._stop_event.clear()
|
||||
|
||||
logger.info(f"Starting agent scheduler: {getattr(self.agent, 'name', 'Agent')}")
|
||||
logger.info(f"Task: {self.task}")
|
||||
logger.info(f"Schedule: {schedule_expr} ({interval}s interval)")
|
||||
logger.debug(f"Starting agent scheduler: {getattr(self.agent, 'name', 'Agent')}")
|
||||
logger.debug(f"Task: {self.task}")
|
||||
logger.debug(f"Schedule: {schedule_expr} ({interval}s interval)")
|
||||
self.is_running = True
|
||||
self._stop_event.clear()
|
||||
self._start_time = datetime.now()
|
||||
|
||||
# Run immediately if requested
|
||||
if run_immediately:
|
||||
logger.info("Running agent immediately before starting schedule...")
|
||||
logger.debug("Running agent immediately before starting schedule...")
|
||||
self._execute_with_retry(max_retries)
|
||||
|
||||
self._thread = threading.Thread(
|
||||
@@ -119,7 +119,7 @@ class AgentScheduler:
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
logger.info("Agent scheduler started successfully")
|
||||
logger.debug("Agent scheduler started successfully")
|
||||
if self.timeout:
|
||||
logger.info(f"Timeout per execution: {self.timeout}s")
|
||||
if self.max_cost:
|
||||
@@ -139,18 +139,18 @@ class AgentScheduler:
|
||||
True if stopped successfully
|
||||
"""
|
||||
if not self.is_running:
|
||||
logger.info("Scheduler is not running")
|
||||
logger.debug("Scheduler is not running")
|
||||
return True
|
||||
|
||||
logger.info("Stopping agent scheduler...")
|
||||
logger.debug("Stopping agent scheduler...")
|
||||
self._stop_event.set()
|
||||
|
||||
if self._thread and self._thread.is_alive():
|
||||
self._thread.join(timeout=10)
|
||||
|
||||
self.is_running = False
|
||||
logger.info("Agent scheduler stopped")
|
||||
logger.info(f"Execution stats - Total: {self._execution_count}, Success: {self._success_count}, Failed: {self._failure_count}")
|
||||
logger.debug("Agent scheduler stopped")
|
||||
logger.debug(f"Execution stats - Total: {self._execution_count}, Success: {self._success_count}, Failed: {self._failure_count}")
|
||||
return True
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
@@ -172,6 +172,43 @@ class AgentScheduler:
|
||||
"cost_per_execution": round(self._total_cost / self._execution_count, 4) if self._execution_count > 0 else 0
|
||||
}
|
||||
|
||||
def _update_state_if_daemon(self):
|
||||
"""Update state file with execution stats if running as daemon."""
|
||||
try:
|
||||
import os
|
||||
# Check if we're running as a daemon by looking for state file
|
||||
state_dir = os.path.expanduser("~/.praisonai/schedulers")
|
||||
if not os.path.exists(state_dir):
|
||||
return
|
||||
|
||||
# Try to find our state file by checking all state files for matching PID
|
||||
current_pid = os.getpid()
|
||||
for state_file in os.listdir(state_dir):
|
||||
if not state_file.endswith('.json'):
|
||||
continue
|
||||
|
||||
state_path = os.path.join(state_dir, state_file)
|
||||
try:
|
||||
import json
|
||||
with open(state_path, 'r') as f:
|
||||
state = json.load(f)
|
||||
|
||||
# Check if this is our state file
|
||||
if state.get('pid') == current_pid:
|
||||
# Update execution stats
|
||||
state['executions'] = self._execution_count
|
||||
state['cost'] = round(self._total_cost, 4)
|
||||
|
||||
# Write back
|
||||
with open(state_path, 'w') as f:
|
||||
json.dump(state, f, indent=2)
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
except Exception as e:
|
||||
# Silently fail - don't break scheduler if state update fails
|
||||
logger.debug(f"Failed to update state: {e}")
|
||||
|
||||
def _run_schedule(self, interval: int, max_retries: int):
|
||||
"""Internal method to run scheduled agent executions."""
|
||||
while not self._stop_event.is_set():
|
||||
@@ -182,15 +219,15 @@ class AgentScheduler:
|
||||
self.stop()
|
||||
break
|
||||
|
||||
logger.info(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Starting scheduled agent execution")
|
||||
logger.debug(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Starting scheduled agent execution")
|
||||
|
||||
self._execute_with_retry(max_retries)
|
||||
|
||||
# Wait for next scheduled time
|
||||
logger.info(f"Next execution in {interval} seconds ({interval/3600:.1f} hours)")
|
||||
logger.debug(f"Next execution in {interval} seconds ({interval/3600:.1f} hours)")
|
||||
if self.max_cost:
|
||||
remaining = self.max_cost - self._total_cost
|
||||
logger.info(f"Budget remaining: ${remaining:.4f}")
|
||||
logger.debug(f"Budget remaining: ${remaining:.4f}")
|
||||
self._stop_event.wait(interval)
|
||||
|
||||
def _execute_with_retry(self, max_retries: int):
|
||||
@@ -201,7 +238,7 @@ class AgentScheduler:
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
logger.info(f"Attempt {attempt + 1}/{max_retries}")
|
||||
logger.debug(f"Attempt {attempt + 1}/{max_retries}")
|
||||
|
||||
# Execute with timeout if specified
|
||||
if self.timeout:
|
||||
@@ -223,8 +260,8 @@ class AgentScheduler:
|
||||
else:
|
||||
result = self._executor.execute(self.task)
|
||||
|
||||
logger.info(f"Agent execution successful on attempt {attempt + 1}")
|
||||
logger.info(f"Result: {result}")
|
||||
logger.debug(f"Agent execution successful on attempt {attempt + 1}")
|
||||
logger.debug(f"Result: {result}")
|
||||
|
||||
# Always print result to stdout (even in non-verbose mode)
|
||||
print(f"\n✅ Agent Response:\n{result}\n")
|
||||
@@ -232,7 +269,7 @@ class AgentScheduler:
|
||||
# Estimate cost (rough: ~$0.0001 per execution for gpt-4o-mini)
|
||||
estimated_cost = 0.0001 # Base cost estimate
|
||||
self._total_cost += estimated_cost
|
||||
logger.info(f"Estimated cost this run: ${estimated_cost:.4f}, Total: ${self._total_cost:.4f}")
|
||||
logger.debug(f"Estimated cost this run: ${estimated_cost:.4f}, Total: ${self._total_cost:.4f}")
|
||||
|
||||
self._success_count += 1
|
||||
success = True
|
||||
@@ -242,6 +279,9 @@ class AgentScheduler:
|
||||
self.on_success(result)
|
||||
except Exception as e:
|
||||
logger.error(f"Callback error in on_success: {e}")
|
||||
|
||||
# Update state file after successful execution
|
||||
self._update_state_if_daemon()
|
||||
|
||||
break
|
||||
|
||||
@@ -253,7 +293,7 @@ class AgentScheduler:
|
||||
|
||||
if attempt < max_retries - 1:
|
||||
wait_time = 30 * (attempt + 1) # Exponential backoff
|
||||
logger.info(f"Waiting {wait_time}s before retry...")
|
||||
logger.debug(f"Waiting {wait_time}s before retry...")
|
||||
time.sleep(wait_time)
|
||||
|
||||
if not success:
|
||||
@@ -265,6 +305,9 @@ class AgentScheduler:
|
||||
self.on_failure(f"Failed after {max_retries} attempts")
|
||||
except Exception as e:
|
||||
logger.error(f"Callback error in on_failure: {e}")
|
||||
|
||||
# Update state file even after failure
|
||||
self._update_state_if_daemon()
|
||||
|
||||
def execute_once(self) -> Any:
|
||||
"""
|
||||
@@ -273,10 +316,10 @@ class AgentScheduler:
|
||||
Returns:
|
||||
Agent execution result
|
||||
"""
|
||||
logger.info("Executing agent once")
|
||||
logger.debug("Executing agent once")
|
||||
try:
|
||||
result = self._executor.execute(self.task)
|
||||
logger.info(f"One-time execution successful: {result}")
|
||||
logger.debug(f"One-time execution successful: {result}")
|
||||
|
||||
if self.on_success:
|
||||
try:
|
||||
|
||||
@@ -50,14 +50,8 @@ class DaemonManager:
|
||||
"""
|
||||
log_file = self.log_dir / f"{name}.log"
|
||||
|
||||
# Open log file
|
||||
# Open log file - use original working approach
|
||||
with open(log_file, 'a') as log:
|
||||
log.write(f"\n{'='*60}\n")
|
||||
log.write(f"Starting daemon: {name}\n")
|
||||
log.write(f"Task: {task}\n")
|
||||
log.write(f"Interval: {interval}\n")
|
||||
log.write(f"Time: {datetime.now().isoformat()}\n")
|
||||
log.write(f"{'='*60}\n\n")
|
||||
log.flush()
|
||||
|
||||
# Start process as daemon
|
||||
@@ -95,7 +89,7 @@ class DaemonManager:
|
||||
Returns:
|
||||
Process ID
|
||||
"""
|
||||
# Build command
|
||||
# Build command - use CLI without --daemon flag (original working approach)
|
||||
command = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
|
||||
@@ -1 +1 @@
|
||||
__version__ = "2.3.67"
|
||||
__version__ = "2.3.75"
|
||||
@@ -12,7 +12,7 @@ dependencies = [
|
||||
"rich>=13.7",
|
||||
"markdown>=3.5",
|
||||
"pyparsing>=3.0.0",
|
||||
"praisonaiagents>=0.1.21",
|
||||
"praisonaiagents>=0.1.23",
|
||||
"python-dotenv>=0.19.0",
|
||||
"instructor>=1.3.3",
|
||||
"PyYAML>=6.0",
|
||||
|
||||
@@ -0,0 +1,396 @@
|
||||
"""
|
||||
Unit tests for CLI compare feature.
|
||||
|
||||
Tests cover:
|
||||
- CompareHandler initialization
|
||||
- Mode configuration
|
||||
- Comparison execution
|
||||
- Result formatting
|
||||
- Output saving
|
||||
"""
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
class TestCompareHandler:
|
||||
"""Tests for CompareHandler class."""
|
||||
|
||||
def test_init_default_values(self):
|
||||
"""Test default initialization values."""
|
||||
from praisonai.cli.features.compare import CompareHandler
|
||||
|
||||
handler = CompareHandler()
|
||||
assert handler.verbose is False
|
||||
assert handler.results == []
|
||||
|
||||
def test_init_with_verbose(self):
|
||||
"""Test initialization with verbose flag."""
|
||||
from praisonai.cli.features.compare import CompareHandler
|
||||
|
||||
handler = CompareHandler(verbose=True)
|
||||
assert handler.verbose is True
|
||||
|
||||
|
||||
class TestCompareModes:
|
||||
"""Tests for comparison mode definitions."""
|
||||
|
||||
def test_get_mode_config_basic(self):
|
||||
"""Test getting basic mode configuration."""
|
||||
from praisonai.cli.features.compare import get_mode_config
|
||||
|
||||
config = get_mode_config("basic")
|
||||
assert config == {}
|
||||
|
||||
def test_get_mode_config_tools(self):
|
||||
"""Test getting tools mode configuration."""
|
||||
from praisonai.cli.features.compare import get_mode_config
|
||||
|
||||
config = get_mode_config("tools")
|
||||
assert "tools" in config
|
||||
|
||||
def test_get_mode_config_research(self):
|
||||
"""Test getting research mode configuration."""
|
||||
from praisonai.cli.features.compare import get_mode_config
|
||||
|
||||
config = get_mode_config("research")
|
||||
assert config.get("research") is True
|
||||
|
||||
def test_get_mode_config_planning(self):
|
||||
"""Test getting planning mode configuration."""
|
||||
from praisonai.cli.features.compare import get_mode_config
|
||||
|
||||
config = get_mode_config("planning")
|
||||
assert config.get("planning") is True
|
||||
|
||||
def test_get_mode_config_memory(self):
|
||||
"""Test getting memory mode configuration."""
|
||||
from praisonai.cli.features.compare import get_mode_config
|
||||
|
||||
config = get_mode_config("memory")
|
||||
assert config.get("memory") is True
|
||||
|
||||
def test_get_mode_config_router(self):
|
||||
"""Test getting router mode configuration."""
|
||||
from praisonai.cli.features.compare import get_mode_config
|
||||
|
||||
config = get_mode_config("router")
|
||||
assert config.get("router") is True
|
||||
|
||||
def test_get_mode_config_web_search(self):
|
||||
"""Test getting web_search mode configuration."""
|
||||
from praisonai.cli.features.compare import get_mode_config
|
||||
|
||||
config = get_mode_config("web_search")
|
||||
assert config.get("web_search") is True
|
||||
|
||||
def test_get_mode_config_unknown(self):
|
||||
"""Test getting unknown mode returns empty config."""
|
||||
from praisonai.cli.features.compare import get_mode_config
|
||||
|
||||
config = get_mode_config("unknown_mode")
|
||||
assert config == {}
|
||||
|
||||
def test_get_mode_config_with_custom_args(self):
|
||||
"""Test getting mode config with custom args override."""
|
||||
from praisonai.cli.features.compare import get_mode_config
|
||||
|
||||
config = get_mode_config("basic", {"llm": "gpt-4o"})
|
||||
assert config.get("llm") == "gpt-4o"
|
||||
|
||||
def test_list_available_modes(self):
|
||||
"""Test listing all available modes."""
|
||||
from praisonai.cli.features.compare import list_available_modes
|
||||
|
||||
modes = list_available_modes()
|
||||
assert "basic" in modes
|
||||
assert "tools" in modes
|
||||
assert "research" in modes
|
||||
assert "planning" in modes
|
||||
|
||||
|
||||
class TestModeResult:
|
||||
"""Tests for ModeResult dataclass."""
|
||||
|
||||
def test_mode_result_creation(self):
|
||||
"""Test creating a ModeResult."""
|
||||
from praisonai.cli.features.compare import ModeResult
|
||||
|
||||
result = ModeResult(
|
||||
mode="basic",
|
||||
output="Test output",
|
||||
execution_time_ms=1234.5,
|
||||
model_used="gpt-4o-mini"
|
||||
)
|
||||
assert result.mode == "basic"
|
||||
assert result.output == "Test output"
|
||||
assert result.execution_time_ms == 1234.5
|
||||
assert result.model_used == "gpt-4o-mini"
|
||||
|
||||
def test_mode_result_with_tokens(self):
|
||||
"""Test ModeResult with token usage."""
|
||||
from praisonai.cli.features.compare import ModeResult
|
||||
|
||||
result = ModeResult(
|
||||
mode="tools",
|
||||
output="Output",
|
||||
execution_time_ms=2000.0,
|
||||
model_used="gpt-4o-mini",
|
||||
tokens={"input": 100, "output": 200}
|
||||
)
|
||||
assert result.tokens["input"] == 100
|
||||
assert result.tokens["output"] == 200
|
||||
|
||||
def test_mode_result_with_tools(self):
|
||||
"""Test ModeResult with tools used."""
|
||||
from praisonai.cli.features.compare import ModeResult
|
||||
|
||||
result = ModeResult(
|
||||
mode="tools",
|
||||
output="Output",
|
||||
execution_time_ms=2000.0,
|
||||
model_used="gpt-4o-mini",
|
||||
tools_used=["internet_search", "calculator"]
|
||||
)
|
||||
assert "internet_search" in result.tools_used
|
||||
assert "calculator" in result.tools_used
|
||||
|
||||
def test_mode_result_to_dict(self):
|
||||
"""Test converting ModeResult to dictionary."""
|
||||
from praisonai.cli.features.compare import ModeResult
|
||||
|
||||
result = ModeResult(
|
||||
mode="basic",
|
||||
output="Test",
|
||||
execution_time_ms=1000.0,
|
||||
model_used="gpt-4o-mini"
|
||||
)
|
||||
d = result.to_dict()
|
||||
assert d["mode"] == "basic"
|
||||
assert d["execution_time_ms"] == 1000.0
|
||||
|
||||
|
||||
class TestCompareResult:
|
||||
"""Tests for CompareResult dataclass."""
|
||||
|
||||
def test_compare_result_creation(self):
|
||||
"""Test creating a CompareResult."""
|
||||
from praisonai.cli.features.compare import CompareResult, ModeResult
|
||||
|
||||
result = CompareResult(
|
||||
query="What is AI?",
|
||||
comparisons=[
|
||||
ModeResult(mode="basic", output="AI is...", execution_time_ms=1000, model_used="gpt-4o-mini"),
|
||||
ModeResult(mode="tools", output="Based on search...", execution_time_ms=2000, model_used="gpt-4o-mini"),
|
||||
]
|
||||
)
|
||||
assert result.query == "What is AI?"
|
||||
assert len(result.comparisons) == 2
|
||||
|
||||
def test_compare_result_summary(self):
|
||||
"""Test CompareResult summary generation."""
|
||||
from praisonai.cli.features.compare import CompareResult, ModeResult
|
||||
|
||||
result = CompareResult(
|
||||
query="Test",
|
||||
comparisons=[
|
||||
ModeResult(mode="basic", output="A", execution_time_ms=1000, model_used="gpt-4o-mini"),
|
||||
ModeResult(mode="tools", output="B", execution_time_ms=2000, model_used="gpt-4o-mini"),
|
||||
]
|
||||
)
|
||||
summary = result.get_summary()
|
||||
assert summary["fastest"] == "basic"
|
||||
assert summary["slowest"] == "tools"
|
||||
|
||||
def test_compare_result_to_dict(self):
|
||||
"""Test converting CompareResult to dictionary."""
|
||||
from praisonai.cli.features.compare import CompareResult, ModeResult
|
||||
|
||||
result = CompareResult(
|
||||
query="Test",
|
||||
comparisons=[
|
||||
ModeResult(mode="basic", output="A", execution_time_ms=1000, model_used="gpt-4o-mini"),
|
||||
]
|
||||
)
|
||||
d = result.to_dict()
|
||||
assert d["query"] == "Test"
|
||||
assert "comparisons" in d
|
||||
assert "summary" in d
|
||||
|
||||
def test_compare_result_to_json(self):
|
||||
"""Test converting CompareResult to JSON."""
|
||||
from praisonai.cli.features.compare import CompareResult, ModeResult
|
||||
|
||||
result = CompareResult(
|
||||
query="Test",
|
||||
comparisons=[
|
||||
ModeResult(mode="basic", output="A", execution_time_ms=1000, model_used="gpt-4o-mini"),
|
||||
]
|
||||
)
|
||||
json_str = result.to_json()
|
||||
data = json.loads(json_str)
|
||||
assert data["query"] == "Test"
|
||||
|
||||
|
||||
class TestCompareExecution:
|
||||
"""Tests for comparison execution."""
|
||||
|
||||
@patch('praisonai.cli.features.compare.CompareHandler._run_mode')
|
||||
def test_compare_single_mode(self, mock_run_mode):
|
||||
"""Test comparing with single mode."""
|
||||
from praisonai.cli.features.compare import CompareHandler, ModeResult
|
||||
|
||||
mock_run_mode.return_value = ModeResult(
|
||||
mode="basic",
|
||||
output="Test output",
|
||||
execution_time_ms=1000,
|
||||
model_used="gpt-4o-mini"
|
||||
)
|
||||
|
||||
handler = CompareHandler()
|
||||
result = handler.compare("What is AI?", modes=["basic"])
|
||||
|
||||
assert len(result.comparisons) == 1
|
||||
assert result.comparisons[0].mode == "basic"
|
||||
|
||||
@patch('praisonai.cli.features.compare.CompareHandler._run_mode')
|
||||
def test_compare_multiple_modes(self, mock_run_mode):
|
||||
"""Test comparing with multiple modes."""
|
||||
from praisonai.cli.features.compare import CompareHandler, ModeResult
|
||||
|
||||
def side_effect(query, mode, **kwargs):
|
||||
return ModeResult(
|
||||
mode=mode,
|
||||
output=f"Output for {mode}",
|
||||
execution_time_ms=1000 if mode == "basic" else 2000,
|
||||
model_used="gpt-4o-mini"
|
||||
)
|
||||
|
||||
mock_run_mode.side_effect = side_effect
|
||||
|
||||
handler = CompareHandler()
|
||||
result = handler.compare("What is AI?", modes=["basic", "tools"])
|
||||
|
||||
assert len(result.comparisons) == 2
|
||||
assert result.comparisons[0].mode == "basic"
|
||||
assert result.comparisons[1].mode == "tools"
|
||||
|
||||
@patch('praisonai.cli.features.compare.CompareHandler._run_mode')
|
||||
def test_compare_with_model_override(self, mock_run_mode):
|
||||
"""Test comparing with model override."""
|
||||
from praisonai.cli.features.compare import CompareHandler, ModeResult
|
||||
|
||||
mock_run_mode.return_value = ModeResult(
|
||||
mode="basic",
|
||||
output="Test",
|
||||
execution_time_ms=1000,
|
||||
model_used="gpt-4o"
|
||||
)
|
||||
|
||||
handler = CompareHandler()
|
||||
compare_result = handler.compare("Test", modes=["basic"], model="gpt-4o")
|
||||
|
||||
mock_run_mode.assert_called_once()
|
||||
call_kwargs = mock_run_mode.call_args[1]
|
||||
assert call_kwargs.get("model") == "gpt-4o"
|
||||
assert compare_result.query == "Test"
|
||||
|
||||
|
||||
class TestCompareOutput:
|
||||
"""Tests for comparison output formatting and saving."""
|
||||
|
||||
def test_save_result_to_file(self):
|
||||
"""Test saving comparison result to file."""
|
||||
from praisonai.cli.features.compare import CompareResult, ModeResult, save_compare_result
|
||||
|
||||
result = CompareResult(
|
||||
query="Test",
|
||||
comparisons=[
|
||||
ModeResult(mode="basic", output="A", execution_time_ms=1000, model_used="gpt-4o-mini"),
|
||||
]
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/result.json"
|
||||
success = save_compare_result(result, path)
|
||||
|
||||
assert success is True
|
||||
assert Path(path).exists()
|
||||
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
assert data["query"] == "Test"
|
||||
|
||||
def test_format_comparison_table(self):
|
||||
"""Test formatting comparison as table."""
|
||||
from praisonai.cli.features.compare import CompareResult, ModeResult, format_comparison_table
|
||||
|
||||
result = CompareResult(
|
||||
query="Test",
|
||||
comparisons=[
|
||||
ModeResult(mode="basic", output="A", execution_time_ms=1000, model_used="gpt-4o-mini"),
|
||||
ModeResult(mode="tools", output="B", execution_time_ms=2000, model_used="gpt-4o-mini"),
|
||||
]
|
||||
)
|
||||
|
||||
table_str = format_comparison_table(result)
|
||||
assert "basic" in table_str
|
||||
assert "tools" in table_str
|
||||
|
||||
|
||||
class TestParseModes:
|
||||
"""Tests for mode parsing from CLI input."""
|
||||
|
||||
def test_parse_modes_comma_separated(self):
|
||||
"""Test parsing comma-separated modes."""
|
||||
from praisonai.cli.features.compare import parse_modes
|
||||
|
||||
modes = parse_modes("basic,tools,research")
|
||||
assert modes == ["basic", "tools", "research"]
|
||||
|
||||
def test_parse_modes_single(self):
|
||||
"""Test parsing single mode."""
|
||||
from praisonai.cli.features.compare import parse_modes
|
||||
|
||||
modes = parse_modes("basic")
|
||||
assert modes == ["basic"]
|
||||
|
||||
def test_parse_modes_with_spaces(self):
|
||||
"""Test parsing modes with spaces."""
|
||||
from praisonai.cli.features.compare import parse_modes
|
||||
|
||||
modes = parse_modes("basic, tools, research")
|
||||
assert modes == ["basic", "tools", "research"]
|
||||
|
||||
def test_parse_modes_empty(self):
|
||||
"""Test parsing empty string returns default."""
|
||||
from praisonai.cli.features.compare import parse_modes
|
||||
|
||||
modes = parse_modes("")
|
||||
assert modes == ["basic"]
|
||||
|
||||
|
||||
class TestCompareWithModel:
|
||||
"""Tests for comparison with model override."""
|
||||
|
||||
@patch('praisonai.cli.features.compare.CompareHandler._run_mode')
|
||||
def test_compare_execute_with_output_path(self, mock_run_mode):
|
||||
"""Test execute method with output path."""
|
||||
from praisonai.cli.features.compare import CompareHandler, ModeResult
|
||||
|
||||
mock_run_mode.return_value = ModeResult(
|
||||
mode="basic",
|
||||
output="Test output",
|
||||
execution_time_ms=1000,
|
||||
model_used="gpt-4o-mini"
|
||||
)
|
||||
|
||||
handler = CompareHandler()
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
output_path = f"{tmpdir}/result.json"
|
||||
compare_result = handler.execute("Test query", "basic", output_path=output_path)
|
||||
|
||||
assert Path(output_path).exists()
|
||||
assert compare_result.query == "Test query"
|
||||
@@ -0,0 +1,555 @@
|
||||
"""
|
||||
Unit tests for PraisonAI Advanced Profiler features.
|
||||
|
||||
TDD: Tests written first for new profiling capabilities:
|
||||
- API/HTTP call profiling (wall-clock time)
|
||||
- Streaming profiling (TTFT, total time)
|
||||
- Line-level profiling
|
||||
- Memory profiling
|
||||
- Flamegraph generation
|
||||
- cProfile integration
|
||||
- Statistics (p50, p95, p99)
|
||||
"""
|
||||
|
||||
import time
|
||||
import asyncio
|
||||
import pytest
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# API Call Profiling Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestAPICallProfiler:
|
||||
"""Test API/HTTP call profiling with wall-clock time."""
|
||||
|
||||
def test_api_call_record(self):
|
||||
"""Should record API call timing."""
|
||||
from praisonai.profiler import Profiler, APICallRecord
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
Profiler.record_api_call(
|
||||
endpoint="https://api.openai.com/v1/chat/completions",
|
||||
method="POST",
|
||||
duration_ms=1500.0,
|
||||
status_code=200,
|
||||
request_size=1024,
|
||||
response_size=2048
|
||||
)
|
||||
|
||||
calls = Profiler.get_api_calls()
|
||||
assert len(calls) >= 1
|
||||
assert calls[-1].endpoint == "https://api.openai.com/v1/chat/completions"
|
||||
assert calls[-1].duration_ms == 1500.0
|
||||
assert calls[-1].status_code == 200
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_api_call_context_manager(self):
|
||||
"""Should profile API call with context manager."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
with Profiler.api_call("https://api.example.com/test", method="GET") as call:
|
||||
time.sleep(0.01) # Simulate API latency
|
||||
|
||||
calls = Profiler.get_api_calls()
|
||||
assert len(calls) >= 1
|
||||
assert calls[-1].duration_ms >= 10
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_api_call_decorator(self):
|
||||
"""Should profile function as API call."""
|
||||
from praisonai.profiler import Profiler, profile_api
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
@profile_api(endpoint="test_endpoint")
|
||||
def mock_api_call():
|
||||
time.sleep(0.01)
|
||||
return {"status": "ok"}
|
||||
|
||||
result = mock_api_call()
|
||||
assert result == {"status": "ok"}
|
||||
|
||||
calls = Profiler.get_api_calls()
|
||||
assert len(calls) >= 1
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Streaming Profiling Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestStreamingProfiler:
|
||||
"""Test streaming profiling (TTFT, chunk times, total time)."""
|
||||
|
||||
def test_streaming_record(self):
|
||||
"""Should record streaming metrics."""
|
||||
from praisonai.profiler import Profiler, StreamingRecord
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
Profiler.record_streaming(
|
||||
name="chat_completion",
|
||||
ttft_ms=150.0, # Time to first token
|
||||
total_ms=2000.0,
|
||||
chunk_count=50,
|
||||
total_tokens=500
|
||||
)
|
||||
|
||||
streams = Profiler.get_streaming_records()
|
||||
assert len(streams) >= 1
|
||||
assert streams[-1].ttft_ms == 150.0
|
||||
assert streams[-1].total_ms == 2000.0
|
||||
assert streams[-1].chunk_count == 50
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_streaming_tracker(self):
|
||||
"""Should track streaming with context manager."""
|
||||
from praisonai.profiler import Profiler, StreamingTracker
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
tracker = StreamingTracker("test_stream")
|
||||
tracker.start()
|
||||
time.sleep(0.01)
|
||||
tracker.first_token() # Mark TTFT
|
||||
time.sleep(0.02)
|
||||
tracker.chunk() # Record chunk
|
||||
tracker.chunk()
|
||||
tracker.end(total_tokens=100)
|
||||
|
||||
streams = Profiler.get_streaming_records()
|
||||
assert len(streams) >= 1
|
||||
assert streams[-1].ttft_ms >= 10
|
||||
assert streams[-1].chunk_count == 2
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_streaming_context_manager(self):
|
||||
"""Should profile streaming with context manager."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
with Profiler.streaming("test_stream") as tracker:
|
||||
time.sleep(0.01)
|
||||
tracker.first_token()
|
||||
for _ in range(3):
|
||||
tracker.chunk()
|
||||
time.sleep(0.005)
|
||||
|
||||
streams = Profiler.get_streaming_records()
|
||||
assert len(streams) >= 1
|
||||
assert streams[-1].chunk_count == 3
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Memory Profiling Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestMemoryProfiler:
|
||||
"""Test memory profiling capabilities."""
|
||||
|
||||
def test_memory_record(self):
|
||||
"""Should record memory usage."""
|
||||
from praisonai.profiler import Profiler, MemoryRecord
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
Profiler.record_memory(
|
||||
name="agent_init",
|
||||
current_kb=1024.0,
|
||||
peak_kb=2048.0
|
||||
)
|
||||
|
||||
memories = Profiler.get_memory_records()
|
||||
assert len(memories) >= 1
|
||||
assert memories[-1].current_kb == 1024.0
|
||||
assert memories[-1].peak_kb == 2048.0
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_memory_context_manager(self):
|
||||
"""Should profile memory with context manager."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
with Profiler.memory("test_operation"):
|
||||
# Allocate some memory
|
||||
data = [i for i in range(10000)]
|
||||
|
||||
memories = Profiler.get_memory_records()
|
||||
assert len(memories) >= 1
|
||||
assert memories[-1].peak_kb > 0
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_memory_snapshot(self):
|
||||
"""Should take memory snapshot."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
|
||||
snapshot = Profiler.memory_snapshot()
|
||||
assert 'current_kb' in snapshot
|
||||
assert 'peak_kb' in snapshot
|
||||
|
||||
Profiler.disable()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Statistics Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestProfilerStatistics:
|
||||
"""Test statistical analysis of profiling data."""
|
||||
|
||||
def test_percentiles(self):
|
||||
"""Should calculate percentiles (p50, p95, p99)."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
# Add multiple timing records
|
||||
for i in range(100):
|
||||
Profiler.record_timing(f"op_{i}", float(i), "test")
|
||||
|
||||
stats = Profiler.get_statistics(category="test")
|
||||
|
||||
assert 'p50' in stats
|
||||
assert 'p95' in stats
|
||||
assert 'p99' in stats
|
||||
assert 'mean' in stats
|
||||
assert 'std_dev' in stats
|
||||
assert 'min' in stats
|
||||
assert 'max' in stats
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_statistics_by_category(self):
|
||||
"""Should calculate statistics per category."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
for i in range(10):
|
||||
Profiler.record_timing(f"api_{i}", float(i * 100), "api")
|
||||
Profiler.record_timing(f"func_{i}", float(i * 10), "function")
|
||||
|
||||
api_stats = Profiler.get_statistics(category="api")
|
||||
func_stats = Profiler.get_statistics(category="function")
|
||||
|
||||
assert api_stats['mean'] > func_stats['mean']
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# cProfile Integration Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestCProfileIntegration:
|
||||
"""Test cProfile integration for detailed function profiling."""
|
||||
|
||||
def test_cprofile_context_manager(self):
|
||||
"""Should profile with cProfile."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
|
||||
with Profiler.cprofile("test_operation") as stats:
|
||||
# Do some work
|
||||
result = sum(i * i for i in range(1000))
|
||||
|
||||
assert stats is not None
|
||||
assert hasattr(stats, 'total_calls') or hasattr(stats, 'stats')
|
||||
|
||||
Profiler.disable()
|
||||
|
||||
def test_cprofile_decorator(self):
|
||||
"""Should profile function with cProfile decorator."""
|
||||
from praisonai.profiler import profile_detailed
|
||||
|
||||
@profile_detailed
|
||||
def compute_heavy():
|
||||
return sum(i * i for i in range(10000))
|
||||
|
||||
result = compute_heavy()
|
||||
assert result > 0
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Flamegraph Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestFlamegraph:
|
||||
"""Test flamegraph generation."""
|
||||
|
||||
def test_flamegraph_data_generation(self):
|
||||
"""Should generate flamegraph-compatible data."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
# Record some nested operations
|
||||
with Profiler.block("outer"):
|
||||
time.sleep(0.01)
|
||||
with Profiler.block("inner"):
|
||||
time.sleep(0.01)
|
||||
|
||||
flamegraph_data = Profiler.get_flamegraph_data()
|
||||
|
||||
assert flamegraph_data is not None
|
||||
assert isinstance(flamegraph_data, (str, list, dict))
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_flamegraph_export(self):
|
||||
"""Should export flamegraph to file."""
|
||||
import tempfile
|
||||
import os
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
with Profiler.block("test_op"):
|
||||
time.sleep(0.01)
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.svg', delete=False) as f:
|
||||
filepath = f.name
|
||||
|
||||
try:
|
||||
Profiler.export_flamegraph(filepath)
|
||||
assert os.path.exists(filepath)
|
||||
finally:
|
||||
if os.path.exists(filepath):
|
||||
os.unlink(filepath)
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Line-Level Profiling Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestLineProfiler:
|
||||
"""Test line-level profiling integration."""
|
||||
|
||||
def test_line_profile_decorator(self):
|
||||
"""Should profile function line by line."""
|
||||
from praisonai.profiler import profile_lines, Profiler
|
||||
|
||||
Profiler.enable()
|
||||
|
||||
@profile_lines
|
||||
def sample_function():
|
||||
a = 1
|
||||
b = 2
|
||||
c = a + b
|
||||
return c
|
||||
|
||||
result = sample_function()
|
||||
assert result == 3
|
||||
|
||||
# Line profiling data should be available
|
||||
line_data = Profiler.get_line_profile_data()
|
||||
# May be empty if line_profiler not installed, but shouldn't error
|
||||
assert line_data is not None
|
||||
|
||||
Profiler.disable()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Report Export Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestReportExport:
|
||||
"""Test report export in various formats."""
|
||||
|
||||
def test_json_export(self):
|
||||
"""Should export report as JSON."""
|
||||
import json
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
Profiler.record_timing("test_op", 100.0, "test")
|
||||
|
||||
json_report = Profiler.export_json()
|
||||
data = json.loads(json_report)
|
||||
|
||||
assert 'timings' in data or 'summary' in data
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_html_export(self):
|
||||
"""Should export report as HTML."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
Profiler.record_timing("test_op", 100.0, "test")
|
||||
|
||||
html_report = Profiler.export_html()
|
||||
|
||||
assert '<html>' in html_report or '<!DOCTYPE' in html_report.lower()
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_file_export(self):
|
||||
"""Should export report to file."""
|
||||
import tempfile
|
||||
import os
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
Profiler.record_timing("test_op", 100.0, "test")
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
|
||||
filepath = f.name
|
||||
|
||||
try:
|
||||
Profiler.export_to_file(filepath, format="json")
|
||||
assert os.path.exists(filepath)
|
||||
with open(filepath) as f:
|
||||
content = f.read()
|
||||
assert len(content) > 0
|
||||
finally:
|
||||
if os.path.exists(filepath):
|
||||
os.unlink(filepath)
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Zero Performance Impact Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestZeroPerformanceImpact:
|
||||
"""Test that profiling has zero impact when disabled."""
|
||||
|
||||
def test_disabled_no_overhead(self):
|
||||
"""Should have minimal overhead when disabled."""
|
||||
from praisonai.profiler import Profiler, profile
|
||||
|
||||
Profiler.disable()
|
||||
|
||||
@profile
|
||||
def fast_function():
|
||||
return 1 + 1
|
||||
|
||||
# Measure overhead
|
||||
iterations = 10000
|
||||
start = time.perf_counter()
|
||||
for _ in range(iterations):
|
||||
fast_function()
|
||||
duration = time.perf_counter() - start
|
||||
|
||||
# Should complete very fast (< 1ms per call on average)
|
||||
avg_us = (duration / iterations) * 1_000_000
|
||||
assert avg_us < 100 # Less than 100 microseconds per call
|
||||
|
||||
def test_no_data_when_disabled(self):
|
||||
"""Should not record data when disabled."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
Profiler.record_timing("test", 100.0, "test")
|
||||
Profiler.record_api_call("endpoint", "GET", 100.0)
|
||||
|
||||
assert len(Profiler.get_timings()) == 0
|
||||
assert len(Profiler.get_api_calls()) == 0
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Async Profiling Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestAsyncProfiling:
|
||||
"""Test async function profiling."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_api_call(self):
|
||||
"""Should profile async API calls."""
|
||||
from praisonai.profiler import Profiler, profile_api_async
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
@profile_api_async(endpoint="async_test")
|
||||
async def async_api_call():
|
||||
await asyncio.sleep(0.01)
|
||||
return {"status": "ok"}
|
||||
|
||||
result = await async_api_call()
|
||||
assert result == {"status": "ok"}
|
||||
|
||||
calls = Profiler.get_api_calls()
|
||||
assert len(calls) >= 1
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_streaming(self):
|
||||
"""Should profile async streaming."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
async with Profiler.streaming_async("async_stream") as tracker:
|
||||
await asyncio.sleep(0.01)
|
||||
tracker.first_token()
|
||||
for _ in range(3):
|
||||
tracker.chunk()
|
||||
await asyncio.sleep(0.005)
|
||||
|
||||
streams = Profiler.get_streaming_records()
|
||||
assert len(streams) >= 1
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
@@ -0,0 +1,288 @@
|
||||
"""
|
||||
Tests for new scheduler features: stop-all, stats, and clean logs.
|
||||
"""
|
||||
import pytest
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
|
||||
|
||||
class TestStopAllCommand:
|
||||
"""Test stop-all command functionality."""
|
||||
|
||||
def test_stop_all_with_multiple_schedulers(self):
|
||||
"""Test stopping all schedulers at once."""
|
||||
from praisonai.cli.features.agent_scheduler import AgentSchedulerHandler
|
||||
from praisonai.scheduler.state_manager import SchedulerStateManager
|
||||
from praisonai.scheduler.daemon_manager import DaemonManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
state_dir = Path(tmpdir) / "schedulers"
|
||||
log_dir = Path(tmpdir) / "logs"
|
||||
|
||||
state_manager = SchedulerStateManager(state_dir)
|
||||
daemon_manager = DaemonManager(log_dir)
|
||||
|
||||
# Create mock states for 3 schedulers
|
||||
for i in range(1, 4):
|
||||
state_manager.save_state(f"test-{i}", {
|
||||
"name": f"test-{i}",
|
||||
"pid": 1000 + i,
|
||||
"task": f"Task {i}",
|
||||
"interval": "*/10s",
|
||||
"status": "running"
|
||||
})
|
||||
|
||||
# Mock daemon_manager.stop_daemon to return True
|
||||
with patch.object(daemon_manager, 'stop_daemon', return_value=True):
|
||||
result = AgentSchedulerHandler._handle_stop_all(state_manager, daemon_manager)
|
||||
|
||||
assert result == 0
|
||||
# All states should be deleted
|
||||
assert len(state_manager.list_all()) == 0
|
||||
|
||||
def test_stop_all_with_no_schedulers(self):
|
||||
"""Test stop-all when no schedulers are running."""
|
||||
from praisonai.cli.features.agent_scheduler import AgentSchedulerHandler
|
||||
from praisonai.scheduler.state_manager import SchedulerStateManager
|
||||
from praisonai.scheduler.daemon_manager import DaemonManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
state_dir = Path(tmpdir) / "schedulers"
|
||||
log_dir = Path(tmpdir) / "logs"
|
||||
|
||||
state_manager = SchedulerStateManager(state_dir)
|
||||
daemon_manager = DaemonManager(log_dir)
|
||||
|
||||
result = AgentSchedulerHandler._handle_stop_all(state_manager, daemon_manager)
|
||||
|
||||
assert result == 0
|
||||
|
||||
def test_stop_all_with_failures(self):
|
||||
"""Test stop-all when some schedulers fail to stop."""
|
||||
from praisonai.cli.features.agent_scheduler import AgentSchedulerHandler
|
||||
from praisonai.scheduler.state_manager import SchedulerStateManager
|
||||
from praisonai.scheduler.daemon_manager import DaemonManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
state_dir = Path(tmpdir) / "schedulers"
|
||||
log_dir = Path(tmpdir) / "logs"
|
||||
|
||||
state_manager = SchedulerStateManager(state_dir)
|
||||
daemon_manager = DaemonManager(log_dir)
|
||||
|
||||
# Create mock states
|
||||
state_manager.save_state("test-1", {
|
||||
"name": "test-1",
|
||||
"pid": 1001,
|
||||
"task": "Task 1",
|
||||
"interval": "*/10s",
|
||||
"status": "running"
|
||||
})
|
||||
state_manager.save_state("test-2", {
|
||||
"name": "test-2",
|
||||
"pid": 1002,
|
||||
"task": "Task 2",
|
||||
"interval": "*/10s",
|
||||
"status": "running"
|
||||
})
|
||||
|
||||
# Mock stop_daemon to fail for second scheduler
|
||||
def mock_stop(pid):
|
||||
return pid == 1001
|
||||
|
||||
with patch.object(daemon_manager, 'stop_daemon', side_effect=mock_stop):
|
||||
result = AgentSchedulerHandler._handle_stop_all(state_manager, daemon_manager)
|
||||
|
||||
assert result == 1 # Should return 1 when failures occur
|
||||
|
||||
|
||||
class TestStatsCommand:
|
||||
"""Test stats command functionality."""
|
||||
|
||||
def test_stats_aggregate(self):
|
||||
"""Test aggregate stats for all schedulers."""
|
||||
from praisonai.cli.features.agent_scheduler import AgentSchedulerHandler
|
||||
from praisonai.scheduler.state_manager import SchedulerStateManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
state_dir = Path(tmpdir) / "schedulers"
|
||||
state_manager = SchedulerStateManager(state_dir)
|
||||
|
||||
# Create mock states with execution data
|
||||
state_manager.save_state("test-1", {
|
||||
"name": "test-1",
|
||||
"pid": 1001,
|
||||
"task": "Task 1",
|
||||
"interval": "*/10s",
|
||||
"status": "running",
|
||||
"executions": 5,
|
||||
"cost": 0.0005
|
||||
})
|
||||
state_manager.save_state("test-2", {
|
||||
"name": "test-2",
|
||||
"pid": 1002,
|
||||
"task": "Task 2",
|
||||
"interval": "*/20s",
|
||||
"status": "stopped",
|
||||
"executions": 3,
|
||||
"cost": 0.0003
|
||||
})
|
||||
|
||||
result = AgentSchedulerHandler._handle_stats(state_manager)
|
||||
|
||||
assert result == 0
|
||||
|
||||
def test_stats_with_no_schedulers(self):
|
||||
"""Test stats when no schedulers exist."""
|
||||
from praisonai.cli.features.agent_scheduler import AgentSchedulerHandler
|
||||
from praisonai.scheduler.state_manager import SchedulerStateManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
state_dir = Path(tmpdir) / "schedulers"
|
||||
state_manager = SchedulerStateManager(state_dir)
|
||||
|
||||
result = AgentSchedulerHandler._handle_stats(state_manager)
|
||||
|
||||
assert result == 0
|
||||
|
||||
def test_stats_individual_scheduler(self):
|
||||
"""Test stats command as alias for describe."""
|
||||
from praisonai.cli.features.agent_scheduler import AgentSchedulerHandler
|
||||
from praisonai.scheduler.state_manager import SchedulerStateManager
|
||||
from praisonai.scheduler.daemon_manager import DaemonManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
state_dir = Path(tmpdir) / "schedulers"
|
||||
log_dir = Path(tmpdir) / "logs"
|
||||
|
||||
state_manager = SchedulerStateManager(state_dir)
|
||||
daemon_manager = DaemonManager(log_dir)
|
||||
|
||||
# Create mock state
|
||||
state_manager.save_state("test-1", {
|
||||
"name": "test-1",
|
||||
"pid": 1001,
|
||||
"task": "Task 1",
|
||||
"interval": "*/10s",
|
||||
"status": "running",
|
||||
"executions": 5,
|
||||
"cost": 0.0005,
|
||||
"started_at": "2025-12-22T10:00:00"
|
||||
})
|
||||
|
||||
result = AgentSchedulerHandler._handle_stats(
|
||||
state_manager,
|
||||
unknown_args=["test-1"],
|
||||
daemon_manager=daemon_manager
|
||||
)
|
||||
|
||||
assert result == 0
|
||||
|
||||
|
||||
class TestCleanLogs:
|
||||
"""Test clean log output (DEBUG level)."""
|
||||
|
||||
def test_scheduler_uses_debug_logging(self):
|
||||
"""Test that scheduler uses DEBUG level for internal logs."""
|
||||
from praisonai.scheduler.agent_scheduler import AgentScheduler
|
||||
from praisonaiagents import Agent
|
||||
import logging
|
||||
|
||||
# Create a test handler to capture log records
|
||||
test_handler = logging.Handler()
|
||||
test_handler.setLevel(logging.DEBUG)
|
||||
records = []
|
||||
|
||||
def capture_record(record):
|
||||
records.append(record)
|
||||
return True
|
||||
|
||||
test_handler.emit = lambda record: records.append(record)
|
||||
|
||||
# Get the scheduler logger
|
||||
logger = logging.getLogger('praisonai.scheduler')
|
||||
logger.addHandler(test_handler)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
try:
|
||||
agent = Agent(
|
||||
name="Test Agent",
|
||||
role="Tester",
|
||||
goal="Test logging",
|
||||
instructions="Test"
|
||||
)
|
||||
|
||||
scheduler = AgentScheduler(
|
||||
agent=agent,
|
||||
task="Test task",
|
||||
timeout=5,
|
||||
max_cost=1.0
|
||||
)
|
||||
|
||||
# Start and immediately stop
|
||||
scheduler.start(schedule_expr="*/10s", max_retries=1, run_immediately=False)
|
||||
scheduler.stop()
|
||||
|
||||
# Check that debug messages were logged
|
||||
debug_messages = [r for r in records if r.levelno == logging.DEBUG]
|
||||
assert len(debug_messages) > 0, "Should have DEBUG level logs"
|
||||
|
||||
finally:
|
||||
logger.removeHandler(test_handler)
|
||||
|
||||
|
||||
class TestStateUpdatePersistence:
|
||||
"""Test that execution stats are persisted to state files."""
|
||||
|
||||
def test_state_update_after_execution(self):
|
||||
"""Test that _update_state_if_daemon updates the state file."""
|
||||
from praisonai.scheduler.agent_scheduler import AgentScheduler
|
||||
from praisonai.scheduler.state_manager import SchedulerStateManager
|
||||
from praisonaiagents import Agent
|
||||
import os
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
state_dir = Path(tmpdir) / "schedulers"
|
||||
state_dir.mkdir(parents=True)
|
||||
|
||||
# Create a mock state file
|
||||
state_manager = SchedulerStateManager(state_dir)
|
||||
pid = os.getpid()
|
||||
|
||||
state_manager.save_state("test-scheduler", {
|
||||
"name": "test-scheduler",
|
||||
"pid": pid,
|
||||
"task": "Test",
|
||||
"interval": "*/10s",
|
||||
"status": "running",
|
||||
"executions": 0,
|
||||
"cost": 0.0
|
||||
})
|
||||
|
||||
agent = Agent(
|
||||
name="Test Agent",
|
||||
role="Tester",
|
||||
goal="Test",
|
||||
instructions="Test"
|
||||
)
|
||||
|
||||
scheduler = AgentScheduler(
|
||||
agent=agent,
|
||||
task="Test task",
|
||||
timeout=5,
|
||||
max_cost=1.0
|
||||
)
|
||||
|
||||
# Simulate execution
|
||||
scheduler._execution_count = 5
|
||||
scheduler._total_cost = 0.0005
|
||||
|
||||
# Call update method
|
||||
scheduler._update_state_if_daemon()
|
||||
|
||||
# Check if state was updated
|
||||
state = state_manager.load_state("test-scheduler")
|
||||
if state: # State might be updated
|
||||
assert state.get('executions', 0) >= 0
|
||||
assert state.get('cost', 0.0) >= 0.0
|
||||
Generated
+4
-4
@@ -4197,7 +4197,7 @@ requires-dist = [
|
||||
{ name = "praisonai-tools", marker = "extra == 'autogen'", specifier = ">=0.0.22" },
|
||||
{ name = "praisonai-tools", marker = "extra == 'autogen-v4'", specifier = ">=0.0.22" },
|
||||
{ name = "praisonai-tools", marker = "extra == 'crewai'", specifier = ">=0.0.22" },
|
||||
{ name = "praisonaiagents", specifier = ">=0.1.21" },
|
||||
{ name = "praisonaiagents", specifier = ">=0.1.23" },
|
||||
{ name = "pyautogen", marker = "extra == 'autogen'", specifier = "==0.2.29" },
|
||||
{ name = "pydantic", marker = "extra == 'chat'", specifier = "<=2.10.1" },
|
||||
{ name = "pydantic", marker = "extra == 'code'", specifier = "<=2.10.1" },
|
||||
@@ -4252,7 +4252,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "praisonaiagents"
|
||||
version = "0.1.21"
|
||||
version = "0.1.23"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "aiohttp" },
|
||||
@@ -4261,9 +4261,9 @@ dependencies = [
|
||||
{ name = "pydantic" },
|
||||
{ name = "rich" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/44/9a/c5b74fdfafc3557107f5c5dc35978d818537f35047f76e2a50142a2401bd/praisonaiagents-0.1.21.tar.gz", hash = "sha256:09b9448f4aa8c88d2ff3cab6514b5746ff62cbe07db97541a467ac1eac38777f", size = 658730, upload-time = "2025-12-20T11:57:22.359Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/79/5e/563a931f0d79f6c64fa8f891ad6eb88cacbbd3f26af5efe52c07a31f11b4/praisonaiagents-0.1.23.tar.gz", hash = "sha256:7a840172892c67feb68e04167db827017d5df8a0e496eb91e309d0e272db60c9", size = 678076, upload-time = "2025-12-22T17:00:28.27Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/89/0a/c408a92c1721bc7a15e1c6dffc099381411fc87c241fba09d60c91a74d47/praisonaiagents-0.1.21-py3-none-any.whl", hash = "sha256:9d77c484013a1ea7c52ad6a9a0750c0c336133f6e913da6b9bd3da73f2bb5b11", size = 589189, upload-time = "2025-12-20T11:57:20.651Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b4/b3/bf02847078595ab1cf109dfd3c7282eaece21f7b879fe96a3ae0d23e8571/praisonaiagents-0.1.23-py3-none-any.whl", hash = "sha256:a455c8fd04fdbd7dd7bf9755c24f19c9c496c404ef60a4fdfc4d4a51decb2e60", size = 616785, upload-time = "2025-12-22T17:00:26.55Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
Reference in new issue
Block a user