mirror of
https://github.com/MervinPraison/PraisonAI.git
synced 2026-11-07 17:33:33 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
308758cdc2 | ||
|
|
f8fa61b082 | ||
|
|
955d236924 | ||
|
|
d9d63c3a13 | ||
|
|
5bee6eb470 | ||
|
|
7cb1676b7c | ||
|
|
e2e9c3f9ee | ||
|
|
cbe8beeca6 | ||
|
|
f7b66a52b7 | ||
|
|
6dbf194312 | ||
|
|
335e8d7e47 | ||
|
|
d65840e092 | ||
|
|
5a13571fdd | ||
|
|
e3b48c2526 | ||
|
|
8d32238b4c | ||
|
|
c17b1298bb | ||
|
|
ec6c150315 | ||
|
|
3d26cedd5d | ||
|
|
8c735a60b2 | ||
|
|
2f5f2f1c13 | ||
|
|
0253073c68 | ||
|
|
7f0e334e8c | ||
|
|
cacf1097ef | ||
|
|
09f01919d2 | ||
|
|
92397615f8 | ||
|
|
4b21943a53 | ||
|
|
af56a753bd | ||
|
|
15c2559a3c | ||
|
|
3215bdc0a6 | ||
|
|
0ff0d83744 | ||
|
|
8f956f91dd | ||
|
|
954f6ac33d | ||
|
|
264d11d8d9 | ||
|
|
88da2bdd7b | ||
|
|
b84e6c2460 | ||
|
|
0434b23ada | ||
|
|
a9951e0694 | ||
|
|
073062cd96 | ||
|
|
936b422876 | ||
|
|
99ab421ce2 | ||
|
|
c1f334905e |
No files matched your search
@@ -171,12 +171,13 @@ npm install praisonai
|
||||
|
||||
| Feature | Code | Docs |
|
||||
|---------|:----:|:----:|
|
||||
| 🚀 Single Agent | [Example](examples/python/agents/single-agent.py) | [📖](https://docs.praison.ai/agents/single) |
|
||||
| 🤝 Multi Agents | [Example](examples/python/general/mini_agents_example.py) | [📖](https://docs.praison.ai/concepts/agents) |
|
||||
| 🤖 Auto Agents | [Example](examples/python/general/auto_agents_example.py) | [📖](https://docs.praison.ai/features/autoagents) |
|
||||
| 🔄 Self Reflection AI Agents | [Example](examples/python/concepts/self-reflection-details.py) | [📖](https://docs.praison.ai/features/selfreflection) |
|
||||
| 🧠 Reasoning AI Agents | [Example](examples/python/concepts/reasoning-extraction.py) | [📖](https://docs.praison.ai/features/reasoning) |
|
||||
| 👁️ Multi Modal AI Agents | [Example](examples/python/general/multimodal.py) | [📖](https://docs.praison.ai/features/multimodal) |
|
||||
| **🤖 Core Agents** | | |
|
||||
| ↳ Single Agent | [Example](examples/python/agents/single-agent.py) | [📖](https://docs.praison.ai/agents/single) |
|
||||
| ↳ Multi Agents | [Example](examples/python/general/mini_agents_example.py) | [📖](https://docs.praison.ai/concepts/agents) |
|
||||
| ↳ Auto Agents | [Example](examples/python/general/auto_agents_example.py) | [📖](https://docs.praison.ai/features/autoagents) |
|
||||
| ↳ Self Reflection AI Agents | [Example](examples/python/concepts/self-reflection-details.py) | [📖](https://docs.praison.ai/features/selfreflection) |
|
||||
| ↳ Reasoning AI Agents | [Example](examples/python/concepts/reasoning-extraction.py) | [📖](https://docs.praison.ai/features/reasoning) |
|
||||
| ↳ Multi Modal AI Agents | [Example](examples/python/general/multimodal.py) | [📖](https://docs.praison.ai/features/multimodal) |
|
||||
| **🔄 Workflows** | | |
|
||||
| ↳ Simple Workflow | [Example](examples/python/workflows/simple_workflow.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow with Agents | [Example](examples/python/workflows/workflow_with_agents.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
@@ -188,10 +189,7 @@ npm install praisonai
|
||||
| ↳ Workflow Branching | [Example](examples/python/workflows/workflow_branching.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow Early Stop | [Example](examples/python/workflows/workflow_early_stop.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow Checkpoints | [Example](examples/python/workflows/workflow_checkpoints.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| 📚 Add Custom Knowledge | [Example](examples/python/concepts/knowledge-agents.py) | [📖](https://docs.praison.ai/features/knowledge) |
|
||||
| 🧠 Memory (Short & Long Term) | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| 📄 Chat with PDF Agents | [Example](examples/python/concepts/chat-with-pdf.py) | [📖](https://docs.praison.ai/features/chat-with-pdf) |
|
||||
| **💻 Code** | | |
|
||||
| **💻 Code & Development** | | |
|
||||
| ↳ Code Interpreter Agents | [Example](examples/python/agents/code-agent.py) | [📖](https://docs.praison.ai/features/codeagent) |
|
||||
| ↳ AI Code Editing Tools | [Example](examples/python/code/code_editing_example.py) | [📖](https://docs.praison.ai/code/editing) |
|
||||
| ↳ External Agents (All) | [Example](examples/python/code/external_agents_example.py) | [📖](https://docs.praison.ai/code/external-agents) |
|
||||
@@ -199,65 +197,78 @@ npm install praisonai
|
||||
| ↳ Gemini CLI | [Example](examples/python/code/gemini_cli_example.py) | [📖](https://docs.praison.ai/code/gemini-cli) |
|
||||
| ↳ Codex CLI | [Example](examples/python/code/codex_cli_example.py) | [📖](https://docs.praison.ai/code/codex-cli) |
|
||||
| ↳ Cursor CLI | [Example](examples/python/code/cursor_cli_example.py) | [📖](https://docs.praison.ai/code/cursor-cli) |
|
||||
| 📚 RAG Agents | [Example](examples/python/concepts/rag-agents.py) | [📖](https://docs.praison.ai/features/rag) |
|
||||
| 🤔 Async & Parallel Processing | [Example](examples/python/general/async_example.py) | [📖](https://docs.praison.ai/features/async) |
|
||||
| 🔢 Math Agents | [Example](examples/python/agents/math-agent.py) | [📖](https://docs.praison.ai/features/mathagent) |
|
||||
| 🎯 Structured Output Agents | [Example](examples/python/general/structured_agents_example.py) | [📖](https://docs.praison.ai/features/structured) |
|
||||
| 🔗 LangChain Integrated Agents | [Example](examples/python/general/langchain_example.py) | [📖](https://docs.praison.ai/features/langchain) |
|
||||
| 📞 Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
|
||||
| 🛠️ 100+ Custom Tools | [Example](examples/python/general/tools_example.py) | [📖](https://docs.praison.ai/tools/tools) |
|
||||
| 📄 YAML Configuration | [Example](examples/cookbooks/yaml/secondary_market_research_agents.yaml) | [📖](https://docs.praison.ai/developers/agents-playbook) |
|
||||
| 💯 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
|
||||
| 🔬 Deep Research Agents | [Example](examples/python/agents/research-agent.py) | [📖](https://docs.praison.ai/agents/deep-research) |
|
||||
| 🔄 Query Rewriter Agent | [Example](#5-query-rewriter-agent) | [📖](https://docs.praison.ai/agents/query-rewriter) |
|
||||
| 🌐 Native Web Search | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/agents/websearch) |
|
||||
| 📥 Web Fetch (Anthropic) | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| 💾 Prompt Caching | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| 🧠 Claude Memory Tool | [Example](#claude-memory-tool-cli) | [📖](https://docs.praison.ai/features/claude-memory-tool) |
|
||||
| 💾 File-Based Memory | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| 🔍 Built-in Search Tools | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/tools/tavily) |
|
||||
| 🔎 Unified Web Search | [Example](src/praisonai-agents/examples/web_search_example.py) | [📖](https://docs.praison.ai/tools/web-search) |
|
||||
| 📋 Planning Mode | [Example](examples/python/agents/planning-agent.py) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🔧 Planning Tools | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🧠 Planning Reasoning | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🔌 MCP Transports | [Example](examples/python/mcp/mcp-transports-overview.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| 🌐 WebSocket MCP | [Example](examples/python/mcp/websocket-mcp.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| 🔐 MCP Security | [Example](examples/python/mcp/mcp-security.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| 🔄 MCP Resumability | [Example](examples/python/mcp/mcp-resumability.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ⚡ Fast Context | [Example](examples/context/00_agent_fast_context_basic.py) | [📖](https://docs.praison.ai/features/fast-context) |
|
||||
| 🖼️ Image Generation Agent | [Example](examples/python/image/image-agent.py) | [📖](https://docs.praison.ai/features/image-generation) |
|
||||
| 📷 Image to Text Agent | [Example](examples/python/agents/image-to-text-agent.py) | [📖](https://docs.praison.ai/agents/image-to-text) |
|
||||
| 🎬 Video Agent | [Example](examples/python/agents/video-agent.py) | [📖](https://docs.praison.ai/agents/video) |
|
||||
| 📊 Data Analyst Agent | [Example](examples/python/agents/data-analyst-agent.py) | [📖](https://docs.praison.ai/agents/data-analyst) |
|
||||
| 💰 Finance Agent | [Example](examples/python/agents/finance-agent.py) | [📖](https://docs.praison.ai/agents/finance) |
|
||||
| 🛒 Shopping Agent | [Example](examples/python/agents/shopping-agent.py) | [📖](https://docs.praison.ai/agents/shopping) |
|
||||
| ⭐ Recommendation Agent | [Example](examples/python/agents/recommendation-agent.py) | [📖](https://docs.praison.ai/agents/recommendation) |
|
||||
| 📖 Wikipedia Agent | [Example](examples/python/agents/wikipedia-agent.py) | [📖](https://docs.praison.ai/agents/wikipedia) |
|
||||
| 💻 Programming Agent | [Example](examples/python/agents/programming-agent.py) | [📖](https://docs.praison.ai/agents/programming) |
|
||||
| 📝 Markdown Agent | [Example](examples/python/agents/markdown-agent.py) | [📖](https://docs.praison.ai/agents/markdown) |
|
||||
| 📝 Prompt Expander Agent | [Example](#prompt-expansion) | [📖](https://docs.praison.ai/agents/prompt-expander) |
|
||||
| 🔀 Model Router | [Example](examples/python/agents/router-agent-cost-optimization.py) | [📖](https://docs.praison.ai/features/model-router) |
|
||||
| ⛓️ Prompt Chaining | [Example](examples/python/general/prompt_chaining.py) | [📖](https://docs.praison.ai/features/promptchaining) |
|
||||
| 🔍 Evaluator Optimiser | [Example](examples/python/general/evaluator-optimiser.py) | [📖](https://docs.praison.ai/features/evaluator-optimiser) |
|
||||
| 👷 Orchestrator Workers | [Example](examples/python/general/orchestrator-workers.py) | [📖](https://docs.praison.ai/features/orchestrator-worker) |
|
||||
| ⚡ Parallelisation | [Example](examples/python/general/parallelisation.py) | [📖](https://docs.praison.ai/features/parallelisation) |
|
||||
| 🔁 Repetitive Agents | [Example](examples/python/concepts/repetitive-agents.py) | [📖](https://docs.praison.ai/features/repetitive) |
|
||||
| 🤝 Agent Handoffs | [Example](examples/python/handoff/handoff_basic.py) | [📖](https://docs.praison.ai/features/handoffs) |
|
||||
| 🛡️ Guardrails | [Example](examples/python/guardrails/comprehensive-guardrails-example.py) | [📖](https://docs.praison.ai/features/guardrails) |
|
||||
| 💬 Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
|
||||
| ✅ Human Approval | [Example](examples/python/general/human_approval_example.py) | [📖](https://docs.praison.ai/features/approval) |
|
||||
| 🔄 Stateful Agents | [Example](examples/python/stateful/workflow-state-example.py) | [📖](https://docs.praison.ai/features/stateful-agents) |
|
||||
| 🤖 Autonomous Workflow | [Example](examples/python/general/autonomous-agent.py) | [📖](https://docs.praison.ai/features/autonomous-workflow) |
|
||||
| 📜 Rules & Instructions | [Example](#6-rules--instructions) | [📖](https://docs.praison.ai/features/rules) |
|
||||
| 🪝 Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
|
||||
| 📈 Telemetry | [Example](examples/python/telemetry/production-telemetry-example.py) | [📖](https://docs.praison.ai/features/telemetry) |
|
||||
| 📹 Camera Integration | [Example](examples/python/camera/) | [📖](https://docs.praison.ai/features/camera-integration) |
|
||||
| 📄 Project Docs (.praison/docs/) | [Example](#docs-cli) | [📖](https://docs.praison.ai/docs/cli/docs) |
|
||||
| 🔌 MCP Config Management | [Example](#mcp-config-cli) | [📖](https://docs.praison.ai/docs/cli/mcp) |
|
||||
| 💬 AI Commit Messages | [Example](#ai-commit-cli) | [📖](https://docs.praison.ai/docs/cli/commit) |
|
||||
| @ @Mentions in Prompts | [Example](#mentions-in-prompts) | [📖](https://docs.praison.ai/docs/cli/mentions) |
|
||||
| 💾 Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| 📜 History in Context | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| **🧠 Memory & Knowledge** | | |
|
||||
| ↳ Memory (Short & Long Term) | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| ↳ File-Based Memory | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| ↳ Claude Memory Tool | [Example](#claude-memory-tool-cli) | [📖](https://docs.praison.ai/features/claude-memory-tool) |
|
||||
| ↳ Add Custom Knowledge | [Example](examples/python/concepts/knowledge-agents.py) | [📖](https://docs.praison.ai/features/knowledge) |
|
||||
| ↳ RAG Agents | [Example](examples/python/concepts/rag-agents.py) | [📖](https://docs.praison.ai/features/rag) |
|
||||
| ↳ Chat with PDF Agents | [Example](examples/python/concepts/chat-with-pdf.py) | [📖](https://docs.praison.ai/features/chat-with-pdf) |
|
||||
| **🔬 Research & Intelligence** | | |
|
||||
| ↳ Deep Research Agents | [Example](examples/python/agents/research-agent.py) | [📖](https://docs.praison.ai/agents/deep-research) |
|
||||
| ↳ Query Rewriter Agent | [Example](#5-query-rewriter-agent) | [📖](https://docs.praison.ai/agents/query-rewriter) |
|
||||
| ↳ Native Web Search | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/agents/websearch) |
|
||||
| ↳ Built-in Search Tools | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/tools/tavily) |
|
||||
| ↳ Unified Web Search | [Example](src/praisonai-agents/examples/web_search_example.py) | [📖](https://docs.praison.ai/tools/web-search) |
|
||||
| ↳ Web Fetch (Anthropic) | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| **📋 Planning & Execution** | | |
|
||||
| ↳ Planning Mode | [Example](examples/python/agents/planning-agent.py) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Planning Tools | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Planning Reasoning | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Prompt Chaining | [Example](examples/python/general/prompt_chaining.py) | [📖](https://docs.praison.ai/features/promptchaining) |
|
||||
| ↳ Evaluator Optimiser | [Example](examples/python/general/evaluator-optimiser.py) | [📖](https://docs.praison.ai/features/evaluator-optimiser) |
|
||||
| ↳ Orchestrator Workers | [Example](examples/python/general/orchestrator-workers.py) | [📖](https://docs.praison.ai/features/orchestrator-worker) |
|
||||
| **👥 Specialized Agents** | | |
|
||||
| ↳ Data Analyst Agent | [Example](examples/python/agents/data-analyst-agent.py) | [📖](https://docs.praison.ai/agents/data-analyst) |
|
||||
| ↳ Finance Agent | [Example](examples/python/agents/finance-agent.py) | [📖](https://docs.praison.ai/agents/finance) |
|
||||
| ↳ Shopping Agent | [Example](examples/python/agents/shopping-agent.py) | [📖](https://docs.praison.ai/agents/shopping) |
|
||||
| ↳ Recommendation Agent | [Example](examples/python/agents/recommendation-agent.py) | [📖](https://docs.praison.ai/agents/recommendation) |
|
||||
| ↳ Wikipedia Agent | [Example](examples/python/agents/wikipedia-agent.py) | [📖](https://docs.praison.ai/agents/wikipedia) |
|
||||
| ↳ Programming Agent | [Example](examples/python/agents/programming-agent.py) | [📖](https://docs.praison.ai/agents/programming) |
|
||||
| ↳ Math Agents | [Example](examples/python/agents/math-agent.py) | [📖](https://docs.praison.ai/features/mathagent) |
|
||||
| ↳ Markdown Agent | [Example](examples/python/agents/markdown-agent.py) | [📖](https://docs.praison.ai/agents/markdown) |
|
||||
| ↳ Prompt Expander Agent | [Example](#prompt-expansion) | [📖](https://docs.praison.ai/agents/prompt-expander) |
|
||||
| **🎨 Media & Multimodal** | | |
|
||||
| ↳ Image Generation Agent | [Example](examples/python/image/image-agent.py) | [📖](https://docs.praison.ai/features/image-generation) |
|
||||
| ↳ Image to Text Agent | [Example](examples/python/agents/image-to-text-agent.py) | [📖](https://docs.praison.ai/agents/image-to-text) |
|
||||
| ↳ Video Agent | [Example](examples/python/agents/video-agent.py) | [📖](https://docs.praison.ai/agents/video) |
|
||||
| ↳ Camera Integration | [Example](examples/python/camera/) | [📖](https://docs.praison.ai/features/camera-integration) |
|
||||
| **🔌 Protocols & Integration** | | |
|
||||
| ↳ MCP Transports | [Example](examples/python/mcp/mcp-transports-overview.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| ↳ WebSocket MCP | [Example](examples/python/mcp/websocket-mcp.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ↳ MCP Security | [Example](examples/python/mcp/mcp-security.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| ↳ MCP Resumability | [Example](examples/python/mcp/mcp-resumability.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ↳ MCP Config Management | [Example](#mcp-config-cli) | [📖](https://docs.praison.ai/docs/cli/mcp) |
|
||||
| ↳ LangChain Integrated Agents | [Example](examples/python/general/langchain_example.py) | [📖](https://docs.praison.ai/features/langchain) |
|
||||
| **🛡️ Safety & Control** | | |
|
||||
| ↳ Guardrails | [Example](examples/python/guardrails/comprehensive-guardrails-example.py) | [📖](https://docs.praison.ai/features/guardrails) |
|
||||
| ↳ Human Approval | [Example](examples/python/general/human_approval_example.py) | [📖](https://docs.praison.ai/features/approval) |
|
||||
| ↳ Rules & Instructions | [Example](#6-rules--instructions) | [📖](https://docs.praison.ai/features/rules) |
|
||||
| **⚙️ Advanced Features** | | |
|
||||
| ↳ Async & Parallel Processing | [Example](examples/python/general/async_example.py) | [📖](https://docs.praison.ai/features/async) |
|
||||
| ↳ Parallelisation | [Example](examples/python/general/parallelisation.py) | [📖](https://docs.praison.ai/features/parallelisation) |
|
||||
| ↳ Repetitive Agents | [Example](examples/python/concepts/repetitive-agents.py) | [📖](https://docs.praison.ai/features/repetitive) |
|
||||
| ↳ Agent Handoffs | [Example](examples/python/handoff/handoff_basic.py) | [📖](https://docs.praison.ai/features/handoffs) |
|
||||
| ↳ Stateful Agents | [Example](examples/python/stateful/workflow-state-example.py) | [📖](https://docs.praison.ai/features/stateful-agents) |
|
||||
| ↳ Autonomous Workflow | [Example](examples/python/general/autonomous-agent.py) | [📖](https://docs.praison.ai/features/autonomous-workflow) |
|
||||
| ↳ Structured Output Agents | [Example](examples/python/general/structured_agents_example.py) | [📖](https://docs.praison.ai/features/structured) |
|
||||
| ↳ Model Router | [Example](examples/python/agents/router-agent-cost-optimization.py) | [📖](https://docs.praison.ai/features/model-router) |
|
||||
| ↳ Prompt Caching | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| ↳ Fast Context | [Example](examples/context/00_agent_fast_context_basic.py) | [📖](https://docs.praison.ai/features/fast-context) |
|
||||
| **🛠️ Tools & Configuration** | | |
|
||||
| ↳ 100+ Custom Tools | [Example](examples/python/general/tools_example.py) | [📖](https://docs.praison.ai/tools/tools) |
|
||||
| ↳ YAML Configuration | [Example](examples/cookbooks/yaml/secondary_market_research_agents.yaml) | [📖](https://docs.praison.ai/developers/agents-playbook) |
|
||||
| ↳ 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
|
||||
| ↳ Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
|
||||
| ↳ Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
|
||||
| **📊 Monitoring & Management** | | |
|
||||
| ↳ Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
|
||||
| ↳ Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| ↳ History in Context | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| ↳ Telemetry | [Example](examples/python/telemetry/production-telemetry-example.py) | [📖](https://docs.praison.ai/features/telemetry) |
|
||||
| ↳ Project Docs (.praison/docs/) | [Example](#docs-cli) | [📖](https://docs.praison.ai/docs/cli/docs) |
|
||||
| ↳ AI Commit Messages | [Example](#ai-commit-cli) | [📖](https://docs.praison.ai/docs/cli/commit) |
|
||||
| ↳ @Mentions in Prompts | [Example](#mentions-in-prompts) | [📖](https://docs.praison.ai/docs/cli/mentions) |
|
||||
| **🖥️ CLI Features** | | |
|
||||
| ↳ Slash Commands | [Example](examples/python/cli/slash_commands_example.py) | [📖](https://docs.praison.ai/docs/cli/slash-commands) |
|
||||
| ↳ Autonomy Modes | [Example](examples/python/cli/autonomy_modes_example.py) | [📖](https://docs.praison.ai/docs/cli/autonomy-modes) |
|
||||
@@ -266,6 +277,23 @@ npm install praisonai
|
||||
| ↳ Interactive TUI | [Example](examples/python/cli/interactive_tui_example.py) | [📖](https://docs.praison.ai/docs/cli/interactive-tui) |
|
||||
| ↳ Git Integration | [Example](examples/python/cli/git_integration_example.py) | [📖](https://docs.praison.ai/docs/cli/git-integration) |
|
||||
| ↳ Sandbox Execution | [Example](examples/python/cli/sandbox_execution_example.py) | [📖](https://docs.praison.ai/docs/cli/sandbox-execution) |
|
||||
| ↳ CLI Compare | [Example](examples/compare/cli_compare_basic.py) | [📖](https://docs.praison.ai/docs/cli/compare) |
|
||||
| ↳ Profile/Benchmark | [Example](#profile-benchmark) | [📖](https://docs.praison.ai/docs/cli/profile) |
|
||||
| ↳ Auto Mode | [Example](#auto-mode) | [📖](https://docs.praison.ai/docs/cli/auto) |
|
||||
| ↳ Init | [Example](#init) | [📖](https://docs.praison.ai/docs/cli/init) |
|
||||
| ↳ File Input | [Example](#file-input) | [📖](https://docs.praison.ai/docs/cli/file-input) |
|
||||
| ↳ Final Agent | [Example](#final-agent) | [📖](https://docs.praison.ai/docs/cli/final-agent) |
|
||||
| ↳ Max Tokens | [Example](#max-tokens) | [📖](https://docs.praison.ai/docs/cli/max-tokens) |
|
||||
| **🧪 Evaluation** | | |
|
||||
| ↳ Accuracy Evaluation | [Example](examples/eval/accuracy_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Performance Evaluation | [Example](examples/eval/performance_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Reliability Evaluation | [Example](examples/eval/reliability_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Criteria Evaluation | [Example](examples/eval/criteria_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| **🎯 Agent Skills** | | |
|
||||
| ↳ Skills Management | [Example](examples/skills/basic_skill_usage.py) | [📖](https://docs.praison.ai/features/skills) |
|
||||
| ↳ Custom Skills | [Example](examples/skills/custom_skill_example.py) | [📖](https://docs.praison.ai/features/skills) |
|
||||
| **⏰ 24/7 Scheduling** | | |
|
||||
| ↳ Agent Scheduler | [Example](examples/python/scheduled_agents/news_checker_live.py) | [📖](https://docs.praison.ai/docs/cli/scheduler) |
|
||||
|
||||
---
|
||||
|
||||
@@ -959,6 +987,124 @@ Run with the same simple command:
|
||||
praisonai agents.yaml
|
||||
```
|
||||
|
||||
### 12. MCP (Model Context Protocol)
|
||||
|
||||
PraisonAI supports MCP Protocol Revision 2025-11-25 with multiple transports.
|
||||
|
||||
#### MCP Client (Consume MCP Servers)
|
||||
```python
|
||||
from praisonaiagents import Agent, MCP
|
||||
|
||||
# stdio - Local NPX/Python servers
|
||||
agent = Agent(tools=MCP("npx @modelcontextprotocol/server-memory"))
|
||||
|
||||
# Streamable HTTP - Production servers
|
||||
agent = Agent(tools=MCP("https://api.example.com/mcp"))
|
||||
|
||||
# WebSocket - Real-time bidirectional
|
||||
agent = Agent(tools=MCP("wss://api.example.com/mcp", auth_token="token"))
|
||||
|
||||
# SSE (Legacy) - Backward compatibility
|
||||
agent = Agent(tools=MCP("http://localhost:8080/sse"))
|
||||
|
||||
# With environment variables
|
||||
agent = Agent(
|
||||
tools=MCP(
|
||||
command="npx",
|
||||
args=["-y", "@modelcontextprotocol/server-brave-search"],
|
||||
env={"BRAVE_API_KEY": "your-key"}
|
||||
)
|
||||
)
|
||||
|
||||
# Multiple MCP servers + regular functions
|
||||
def my_custom_tool(query: str) -> str:
|
||||
"""Custom tool function."""
|
||||
return f"Result: {query}"
|
||||
|
||||
agent = Agent(
|
||||
name="MultiToolAgent",
|
||||
instructions="Agent with multiple MCP servers",
|
||||
tools=[
|
||||
MCP("uvx mcp-server-time"), # Time tools
|
||||
MCP("npx @modelcontextprotocol/server-memory"), # Memory tools
|
||||
my_custom_tool # Regular function
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
#### MCP Server (Expose Tools as MCP Server)
|
||||
|
||||
Expose your Python functions as MCP tools for Claude Desktop, Cursor, and other MCP clients:
|
||||
|
||||
```python
|
||||
from praisonaiagents.mcp import ToolsMCPServer
|
||||
|
||||
def search_web(query: str, max_results: int = 5) -> dict:
|
||||
"""Search the web for information."""
|
||||
return {"results": [f"Result for {query}"]}
|
||||
|
||||
def calculate(expression: str) -> dict:
|
||||
"""Evaluate a mathematical expression."""
|
||||
return {"result": eval(expression)}
|
||||
|
||||
# Create and run MCP server
|
||||
server = ToolsMCPServer(name="my-tools")
|
||||
server.register_tools([search_web, calculate])
|
||||
server.run() # stdio for Claude Desktop
|
||||
# server.run_sse(host="0.0.0.0", port=8080) # SSE for web clients
|
||||
```
|
||||
|
||||
#### MCP Features
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| Session Management | Automatic Mcp-Session-Id handling |
|
||||
| Protocol Versioning | Mcp-Protocol-Version header |
|
||||
| Resumability | SSE stream recovery via Last-Event-ID |
|
||||
| Security | Origin validation, DNS rebinding prevention |
|
||||
| WebSocket | Auto-reconnect with exponential backoff |
|
||||
|
||||
### 13. A2A (Agent2Agent Protocol)
|
||||
|
||||
PraisonAI supports the [A2A Protocol](https://a2a-protocol.org) for agent-to-agent communication, enabling your agents to be discovered and collaborate with other AI agents.
|
||||
|
||||
#### A2A Server (Expose Agent as A2A Server)
|
||||
```python
|
||||
from praisonaiagents import Agent, A2A
|
||||
from fastapi import FastAPI
|
||||
|
||||
# Create an agent with tools
|
||||
def search_web(query: str) -> str:
|
||||
"""Search the web for information."""
|
||||
return f"Results for: {query}"
|
||||
|
||||
agent = Agent(
|
||||
name="Research Assistant",
|
||||
role="Research Analyst",
|
||||
goal="Help users research topics",
|
||||
tools=[search_web]
|
||||
)
|
||||
|
||||
# Expose as A2A Server
|
||||
a2a = A2A(agent=agent, url="http://localhost:8000/a2a")
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(a2a.get_router())
|
||||
|
||||
# Run: uvicorn app:app --reload
|
||||
# Agent Card: GET /.well-known/agent.json
|
||||
# Status: GET /status
|
||||
```
|
||||
|
||||
#### A2A Features
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| Agent Card | JSON metadata for agent discovery |
|
||||
| Skills Extraction | Auto-generate skills from tools |
|
||||
| Task Management | Stateful task lifecycle |
|
||||
| Streaming | SSE streaming for real-time updates |
|
||||
|
||||
> **Documentation**: [docs.praison.ai/a2a](https://docs.praison.ai/a2a) | **Examples**: [examples/python/a2a](https://github.com/MervinPraison/PraisonAI/tree/main/examples/python/a2a)
|
||||
|
||||
---
|
||||
|
||||
## 🎯 CLI / No-Code Interface
|
||||
@@ -1058,6 +1204,20 @@ praisonai "Complex analysis task" --planning --planning-reasoning
|
||||
praisonai "Task" --planning --auto-approve-plan
|
||||
```
|
||||
|
||||
### Tool Approval CLI:
|
||||
```bash
|
||||
# Auto-approve ALL tool executions (use with caution!)
|
||||
praisonai "run ls command" --trust
|
||||
|
||||
# Auto-approve tools up to a risk level (prompt for higher)
|
||||
# Levels: low, medium, high, critical
|
||||
praisonai "write to file" --approve-level high # Prompts for critical tools only
|
||||
praisonai "task" --approve-level medium # Prompts for high and critical
|
||||
|
||||
# Default behavior (no flags): prompts for all dangerous tools
|
||||
praisonai "run shell command" # Will prompt for approval
|
||||
```
|
||||
|
||||
### Memory CLI:
|
||||
```bash
|
||||
# Enable memory for agent (persists across sessions)
|
||||
@@ -1228,6 +1388,21 @@ praisonai "Analyze this data" --metrics
|
||||
praisonai "Complex task" --metrics --planning
|
||||
```
|
||||
|
||||
### Scheduler CLI:
|
||||
|
||||
```bash
|
||||
praisonai schedule start <name> "task" --interval hourly
|
||||
praisonai schedule list
|
||||
praisonai schedule logs <name> [--follow]
|
||||
praisonai schedule stop <name>
|
||||
praisonai schedule restart <name>
|
||||
praisonai schedule delete <name>
|
||||
praisonai schedule describe <name>
|
||||
praisonai schedule save <name> [file.yaml]
|
||||
praisonai schedule "task" --interval hourly # foreground mode
|
||||
praisonai schedule agents.yaml # foreground mode
|
||||
```
|
||||
|
||||
### Image Processing CLI:
|
||||
```bash
|
||||
# Process images with vision-based tasks
|
||||
@@ -1679,124 +1854,6 @@ agent = Agent(
|
||||
| Web Fetch | Anthropic |
|
||||
| Prompt Caching | OpenAI (auto), Anthropic, Bedrock, Deepseek |
|
||||
|
||||
## MCP (Model Context Protocol)
|
||||
|
||||
PraisonAI supports MCP Protocol Revision 2025-11-25 with multiple transports.
|
||||
|
||||
### MCP Client (Consume MCP Servers)
|
||||
```python
|
||||
from praisonaiagents import Agent, MCP
|
||||
|
||||
# stdio - Local NPX/Python servers
|
||||
agent = Agent(tools=MCP("npx @modelcontextprotocol/server-memory"))
|
||||
|
||||
# Streamable HTTP - Production servers
|
||||
agent = Agent(tools=MCP("https://api.example.com/mcp"))
|
||||
|
||||
# WebSocket - Real-time bidirectional
|
||||
agent = Agent(tools=MCP("wss://api.example.com/mcp", auth_token="token"))
|
||||
|
||||
# SSE (Legacy) - Backward compatibility
|
||||
agent = Agent(tools=MCP("http://localhost:8080/sse"))
|
||||
|
||||
# With environment variables
|
||||
agent = Agent(
|
||||
tools=MCP(
|
||||
command="npx",
|
||||
args=["-y", "@modelcontextprotocol/server-brave-search"],
|
||||
env={"BRAVE_API_KEY": "your-key"}
|
||||
)
|
||||
)
|
||||
|
||||
# Multiple MCP servers + regular functions
|
||||
def my_custom_tool(query: str) -> str:
|
||||
"""Custom tool function."""
|
||||
return f"Result: {query}"
|
||||
|
||||
agent = Agent(
|
||||
name="MultiToolAgent",
|
||||
instructions="Agent with multiple MCP servers",
|
||||
tools=[
|
||||
MCP("uvx mcp-server-time"), # Time tools
|
||||
MCP("npx @modelcontextprotocol/server-memory"), # Memory tools
|
||||
my_custom_tool # Regular function
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### MCP Server (Expose Tools as MCP Server)
|
||||
|
||||
Expose your Python functions as MCP tools for Claude Desktop, Cursor, and other MCP clients:
|
||||
|
||||
```python
|
||||
from praisonaiagents.mcp import ToolsMCPServer
|
||||
|
||||
def search_web(query: str, max_results: int = 5) -> dict:
|
||||
"""Search the web for information."""
|
||||
return {"results": [f"Result for {query}"]}
|
||||
|
||||
def calculate(expression: str) -> dict:
|
||||
"""Evaluate a mathematical expression."""
|
||||
return {"result": eval(expression)}
|
||||
|
||||
# Create and run MCP server
|
||||
server = ToolsMCPServer(name="my-tools")
|
||||
server.register_tools([search_web, calculate])
|
||||
server.run() # stdio for Claude Desktop
|
||||
# server.run_sse(host="0.0.0.0", port=8080) # SSE for web clients
|
||||
```
|
||||
|
||||
### MCP Features
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| Session Management | Automatic Mcp-Session-Id handling |
|
||||
| Protocol Versioning | Mcp-Protocol-Version header |
|
||||
| Resumability | SSE stream recovery via Last-Event-ID |
|
||||
| Security | Origin validation, DNS rebinding prevention |
|
||||
| WebSocket | Auto-reconnect with exponential backoff |
|
||||
|
||||
## A2A (Agent2Agent Protocol)
|
||||
|
||||
PraisonAI supports the [A2A Protocol](https://a2a-protocol.org) for agent-to-agent communication, enabling your agents to be discovered and collaborate with other AI agents.
|
||||
|
||||
### A2A Server (Expose Agent as A2A Server)
|
||||
```python
|
||||
from praisonaiagents import Agent, A2A
|
||||
from fastapi import FastAPI
|
||||
|
||||
# Create an agent with tools
|
||||
def search_web(query: str) -> str:
|
||||
"""Search the web for information."""
|
||||
return f"Results for: {query}"
|
||||
|
||||
agent = Agent(
|
||||
name="Research Assistant",
|
||||
role="Research Analyst",
|
||||
goal="Help users research topics",
|
||||
tools=[search_web]
|
||||
)
|
||||
|
||||
# Expose as A2A Server
|
||||
a2a = A2A(agent=agent, url="http://localhost:8000/a2a")
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(a2a.get_router())
|
||||
|
||||
# Run: uvicorn app:app --reload
|
||||
# Agent Card: GET /.well-known/agent.json
|
||||
# Status: GET /status
|
||||
```
|
||||
|
||||
### A2A Features
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| Agent Card | JSON metadata for agent discovery |
|
||||
| Skills Extraction | Auto-generate skills from tools |
|
||||
| Task Management | Stateful task lifecycle |
|
||||
| Streaming | SSE streaming for real-time updates |
|
||||
|
||||
> **Documentation**: [docs.praison.ai/a2a](https://docs.praison.ai/a2a) | **Examples**: [examples/python/a2a](https://github.com/MervinPraison/PraisonAI/tree/main/examples/python/a2a)
|
||||
|
||||
## CLI Features
|
||||
|
||||
| Feature | Docs |
|
||||
@@ -2399,6 +2456,7 @@ PraisonAI provides zero-dependency persistent memory for agents. For detailed ex
|
||||
- 🤝 **Agent Handoffs** - Transfer context between specialised agents
|
||||
- 🛡️ **Guardrails** - Input/output validation and safety checks
|
||||
- ✅ **Human Approval** - Require human confirmation for critical actions
|
||||
- 🔐 **Tool Approval CLI** - `--trust` (auto-approve all) and `--approve-level` (risk-based approval)
|
||||
- 💬 **Sessions Management** - Isolated conversation contexts
|
||||
- 🔄 **Stateful Agents** - Maintain state across interactions
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=2.3.52" \
|
||||
"praisonai>=2.3.85" \
|
||||
"praisonai[chat]" \
|
||||
"embedchain[github,youtube]"
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=2.3.52" \
|
||||
"praisonai>=2.3.85" \
|
||||
"praisonai[ui]" \
|
||||
"praisonai[chat]" \
|
||||
"praisonai[realtime]" \
|
||||
|
||||
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=2.3.52" \
|
||||
"praisonai>=2.3.85" \
|
||||
"praisonai[ui]" \
|
||||
"praisonai[crewai]"
|
||||
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
"""
|
||||
CLI Compare Example - Advanced Usage
|
||||
|
||||
This example demonstrates advanced comparison features including
|
||||
custom mode configurations and result analysis.
|
||||
"""
|
||||
|
||||
from praisonai.cli.features.compare import (
|
||||
CompareHandler,
|
||||
CompareResult,
|
||||
ModeResult,
|
||||
get_mode_config,
|
||||
save_compare_result,
|
||||
format_comparison_table,
|
||||
)
|
||||
|
||||
# Create custom mode configurations
|
||||
custom_modes = {
|
||||
"fast": {"llm": "gpt-4o-mini"},
|
||||
"accurate": {"llm": "gpt-4o", "planning": True},
|
||||
"research": {"research": True, "web_search": True},
|
||||
}
|
||||
|
||||
# Create handler with verbose output
|
||||
handler = CompareHandler(verbose=True)
|
||||
|
||||
# Example: Manually create comparison results for analysis
|
||||
# (useful for testing without API calls)
|
||||
mock_result = CompareResult(
|
||||
query="What are the latest AI trends?",
|
||||
comparisons=[
|
||||
ModeResult(
|
||||
mode="basic",
|
||||
output="AI trends include LLMs, multimodal models, and agents.",
|
||||
execution_time_ms=1234.5,
|
||||
model_used="gpt-4o-mini",
|
||||
tools_used=[]
|
||||
),
|
||||
ModeResult(
|
||||
mode="tools",
|
||||
output="Based on recent search results, AI trends in 2024 include...",
|
||||
execution_time_ms=2567.8,
|
||||
model_used="gpt-4o-mini",
|
||||
tools_used=["internet_search"]
|
||||
),
|
||||
ModeResult(
|
||||
mode="research",
|
||||
output="Comprehensive research shows that AI trends...",
|
||||
execution_time_ms=5432.1,
|
||||
model_used="gpt-4o-mini",
|
||||
tools_used=["internet_search", "web_scraper"]
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
# Get summary statistics
|
||||
summary = mock_result.get_summary()
|
||||
print("Comparison Summary:")
|
||||
print(f" Fastest mode: {summary['fastest']} ({summary['fastest_time_ms']:.1f}ms)")
|
||||
print(f" Slowest mode: {summary['slowest']} ({summary['slowest_time_ms']:.1f}ms)")
|
||||
|
||||
# Format as table
|
||||
print("\n" + format_comparison_table(mock_result))
|
||||
|
||||
# Convert to JSON for storage
|
||||
print("\nJSON Output:")
|
||||
print(mock_result.to_json())
|
||||
|
||||
# Save to file
|
||||
# save_compare_result(mock_result, "advanced_comparison.json")
|
||||
|
||||
print("\n--- CLI Usage Examples ---")
|
||||
print("""
|
||||
# Basic comparison
|
||||
praisonai "What is AI?" --compare "basic,tools"
|
||||
|
||||
# Compare with specific model
|
||||
praisonai "Explain quantum computing" --compare "basic,planning,research" --model gpt-4o
|
||||
|
||||
# Save results to file
|
||||
praisonai "Latest tech news" --compare "basic,tools,web_search" --compare-output results.json
|
||||
|
||||
# Verbose output
|
||||
praisonai "Write a poem" --compare "basic,planning" --verbose
|
||||
""")
|
||||
@@ -0,0 +1,45 @@
|
||||
"""
|
||||
CLI Compare Example - Basic Usage
|
||||
|
||||
This example demonstrates how to compare different CLI modes programmatically.
|
||||
"""
|
||||
|
||||
from praisonai.cli.features.compare import (
|
||||
CompareHandler,
|
||||
get_mode_config,
|
||||
list_available_modes,
|
||||
parse_modes,
|
||||
)
|
||||
|
||||
# List all available comparison modes
|
||||
print("Available modes:")
|
||||
for mode in list_available_modes():
|
||||
config = get_mode_config(mode)
|
||||
print(f" - {mode}: {config}")
|
||||
|
||||
# Parse modes from comma-separated string
|
||||
modes = parse_modes("basic,tools,planning")
|
||||
print(f"\nParsed modes: {modes}")
|
||||
|
||||
# Create compare handler
|
||||
handler = CompareHandler(verbose=True)
|
||||
|
||||
# Compare modes (this will run actual agent calls)
|
||||
# Uncomment to run:
|
||||
# result = handler.compare(
|
||||
# query="What is artificial intelligence?",
|
||||
# modes=["basic", "tools"],
|
||||
# model="gpt-4o-mini"
|
||||
# )
|
||||
# handler.print_result(result)
|
||||
|
||||
# Save results to file
|
||||
# result = handler.execute(
|
||||
# query="Explain machine learning",
|
||||
# modes_str="basic,planning",
|
||||
# output_path="comparison_results.json"
|
||||
# )
|
||||
|
||||
print("\nTo run comparison via CLI:")
|
||||
print(' praisonai "What is AI?" --compare "basic,tools,planning"')
|
||||
print(' praisonai "What is AI?" --compare "basic,research" --compare-output results.json')
|
||||
@@ -0,0 +1,23 @@
|
||||
# Example: Data Collection Agent with 30-minute Schedule
|
||||
# Collects and analyzes data every 30 minutes
|
||||
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "Data Collector"
|
||||
role: "Data Analysis Specialist"
|
||||
goal: "Collect and analyze market data at regular intervals"
|
||||
instructions: |
|
||||
You are a data analysis specialist focused on collecting and analyzing
|
||||
market trends and metrics. Provide actionable insights from the data.
|
||||
tools:
|
||||
- search_tool
|
||||
verbose: false
|
||||
|
||||
task: "Collect latest market data and identify key trends"
|
||||
|
||||
# Schedule: Every 30 minutes with immediate execution
|
||||
schedule:
|
||||
interval: "*/30m"
|
||||
max_retries: 5
|
||||
run_immediately: true
|
||||
@@ -0,0 +1,53 @@
|
||||
# Example: 24/7 AI News Monitor with Scheduler
|
||||
# This agent runs continuously at regular intervals to monitor AI news
|
||||
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "AI News Monitor"
|
||||
role: "Technology News Analyst"
|
||||
goal: "Monitor and summarize the latest AI developments"
|
||||
instructions: |
|
||||
You are an expert technology news analyst specializing in AI and machine learning.
|
||||
Your job is to search for the latest AI news, identify the most important developments,
|
||||
and provide clear, concise summaries with sources.
|
||||
|
||||
Focus on:
|
||||
- Major AI product launches
|
||||
- Research breakthroughs
|
||||
- Industry trends
|
||||
- Policy and regulation updates
|
||||
tools:
|
||||
- search_tool
|
||||
verbose: true
|
||||
|
||||
# Task to execute on each scheduled run
|
||||
task: |
|
||||
Search for the latest AI news from the past 24 hours.
|
||||
Provide the top 3 most important stories with:
|
||||
1. Headline
|
||||
2. Brief summary (2-3 sentences)
|
||||
3. Source URL
|
||||
4. Why it's significant
|
||||
|
||||
# Schedule configuration
|
||||
schedule:
|
||||
# Interval options:
|
||||
# - "hourly" - Every hour
|
||||
# - "daily" - Every 24 hours
|
||||
# - "*/30m" - Every 30 minutes
|
||||
# - "*/6h" - Every 6 hours
|
||||
# - "3600" - Custom seconds
|
||||
interval: "hourly"
|
||||
|
||||
# Maximum retry attempts if execution fails
|
||||
max_retries: 3
|
||||
|
||||
# Run immediately when scheduler starts (before first scheduled interval)
|
||||
run_immediately: true
|
||||
|
||||
# Optional: Variables for customization
|
||||
variables:
|
||||
news_category: "AI and Machine Learning"
|
||||
region: "Global"
|
||||
time_range: "24 hours"
|
||||
@@ -0,0 +1,69 @@
|
||||
"""
|
||||
Accuracy Evaluation Example
|
||||
|
||||
This example demonstrates how to evaluate agent output accuracy
|
||||
by comparing against expected outputs using LLM-as-judge.
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.eval import AccuracyEvaluator
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
if has_api_key:
|
||||
# Create a simple math tutor agent
|
||||
agent = Agent(
|
||||
instructions="You are a math tutor. Answer math questions concisely with just the number."
|
||||
)
|
||||
|
||||
# Create accuracy evaluator
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text="What is 2 + 2?",
|
||||
expected_output="4",
|
||||
num_iterations=3, # Run 3 times for statistical significance
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Run evaluation
|
||||
result = evaluator.run(print_summary=True)
|
||||
|
||||
# Check results
|
||||
print(f"\nAverage Score: {result.avg_score}/10")
|
||||
print(f"Passed: {result.passed}")
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Using mock evaluation...")
|
||||
|
||||
# Create evaluator without agent for mock testing
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=lambda x: "4", # Mock function
|
||||
input_text="What is 2 + 2?",
|
||||
expected_output="4",
|
||||
num_iterations=1,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Test with mock output
|
||||
try:
|
||||
result = evaluator.run(print_summary=True)
|
||||
print(f"\nMock Average Score: {result.avg_score}/10")
|
||||
print(f"Mock Passed: {result.passed}")
|
||||
except Exception as e:
|
||||
print(f"Mock evaluation failed: {e}")
|
||||
print("This is expected without an API key")
|
||||
|
||||
# You can also evaluate pre-generated outputs
|
||||
print("\n--- Testing pre-generated output evaluation ---")
|
||||
output = "The answer is 4"
|
||||
|
||||
if has_api_key:
|
||||
try:
|
||||
result2 = evaluator.evaluate_output(output, print_summary=True)
|
||||
print(f"Pre-generated output score: {result2.avg_score}/10")
|
||||
except Exception as e:
|
||||
print(f"Pre-generated evaluation failed: {e}")
|
||||
else:
|
||||
print("⚠️ Skipping pre-generated evaluation (no API key)")
|
||||
print("To run full evaluation, set OPENAI_API_KEY environment variable")
|
||||
@@ -0,0 +1,105 @@
|
||||
"""
|
||||
Batch Evaluation Example
|
||||
|
||||
This example demonstrates how to run batch evaluations
|
||||
from a JSON test file.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import tempfile
|
||||
from praisonaiagents.eval import AccuracyEvaluator
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
# Create a test file with multiple test cases
|
||||
test_cases = [
|
||||
{
|
||||
"input": "What is 2 + 2?",
|
||||
"expected": "4"
|
||||
},
|
||||
{
|
||||
"input": "What is the capital of France?",
|
||||
"expected": "Paris"
|
||||
},
|
||||
{
|
||||
"input": "What color is the sky?",
|
||||
"expected": "Blue"
|
||||
}
|
||||
]
|
||||
|
||||
# Save test cases to a temporary file
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
|
||||
json.dump(test_cases, f)
|
||||
test_file = f.name
|
||||
|
||||
print(f"Test file created: {test_file}")
|
||||
print(f"Running {len(test_cases)} test cases...")
|
||||
|
||||
if has_api_key:
|
||||
# You can use the CLI for batch evaluation:
|
||||
# praisonai eval batch --agent agents.yaml --test-file tests.json --batch-type accuracy
|
||||
|
||||
# Or programmatically:
|
||||
from praisonaiagents import Agent
|
||||
|
||||
agent = Agent(
|
||||
instructions="You are a helpful assistant. Answer questions concisely."
|
||||
)
|
||||
|
||||
results = []
|
||||
for i, test_case in enumerate(test_cases):
|
||||
print(f"\nTest {i + 1}: {test_case['input']}")
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text=test_case["input"],
|
||||
expected_output=test_case["expected"],
|
||||
num_iterations=1
|
||||
)
|
||||
|
||||
result = evaluator.run()
|
||||
results.append({
|
||||
"input": test_case["input"],
|
||||
"expected": test_case["expected"],
|
||||
"score": result.avg_score,
|
||||
"passed": result.passed
|
||||
})
|
||||
|
||||
print(f" Score: {result.avg_score}/10")
|
||||
print(f" Passed: {result.passed}")
|
||||
|
||||
# Summary
|
||||
passed = sum(1 for r in results if r["passed"])
|
||||
print(f"\n{'='*50}")
|
||||
print(f"Summary: {passed}/{len(results)} tests passed")
|
||||
print(f"Pass Rate: {passed/len(results):.1%}")
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Running mock batch evaluation...")
|
||||
|
||||
# Mock evaluation without API key
|
||||
results = []
|
||||
for i, test_case in enumerate(test_cases):
|
||||
print(f"\nTest {i + 1}: {test_case['input']}")
|
||||
|
||||
# Mock result (all would fail without API key)
|
||||
mock_score = 0.0
|
||||
mock_passed = False
|
||||
|
||||
results.append({
|
||||
"input": test_case["input"],
|
||||
"expected": test_case["expected"],
|
||||
"score": mock_score,
|
||||
"passed": mock_passed
|
||||
})
|
||||
|
||||
print(f" Score: {mock_score}/10 (mock)")
|
||||
print(f" Passed: {mock_passed} (expected without API key)")
|
||||
|
||||
# Summary
|
||||
passed = sum(1 for r in results if r["passed"])
|
||||
print(f"\n{'='*50}")
|
||||
print(f"Summary: {passed}/{len(results)} tests passed (mock evaluation)")
|
||||
print(f"Pass Rate: {passed/len(results):.1%}")
|
||||
print("\nTo run real batch evaluation, set OPENAI_API_KEY environment variable")
|
||||
@@ -0,0 +1,82 @@
|
||||
"""
|
||||
Criteria Evaluation Example
|
||||
|
||||
This example demonstrates how to evaluate agent outputs
|
||||
against custom criteria using LLM-as-judge.
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.eval import CriteriaEvaluator
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
if has_api_key:
|
||||
print("--- Testing Criteria Evaluation with Agent ---")
|
||||
# Create a customer service agent
|
||||
agent = Agent(
|
||||
instructions="You are a friendly customer service agent. Be helpful and empathetic."
|
||||
)
|
||||
|
||||
# Create criteria evaluator with numeric scoring
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Response is helpful, empathetic, and provides a clear solution",
|
||||
agent=agent,
|
||||
input_text="I'm frustrated because my order hasn't arrived yet.",
|
||||
scoring_type="numeric", # Score 1-10
|
||||
threshold=7.0, # Pass if score >= 7
|
||||
num_iterations=2,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Run evaluation
|
||||
result = evaluator.run(print_summary=True)
|
||||
|
||||
print("\nNumeric Scoring Results:")
|
||||
print(f" Average Score: {result.avg_score}/10")
|
||||
print(f" Pass Rate: {result.pass_rate:.1%}")
|
||||
print(f" Passed: {result.passed}")
|
||||
|
||||
# Binary scoring example (pass/fail)
|
||||
print("\n--- Testing Binary Scoring ---")
|
||||
binary_evaluator = CriteriaEvaluator(
|
||||
criteria="Response does not contain any offensive language",
|
||||
agent=agent,
|
||||
input_text="Tell me a joke",
|
||||
scoring_type="binary", # Pass or Fail
|
||||
verbose=True
|
||||
)
|
||||
|
||||
binary_result = binary_evaluator.run(print_summary=True)
|
||||
|
||||
# With on_fail callback
|
||||
print("\n--- Testing Failure Callback ---")
|
||||
def handle_failure(score):
|
||||
print(f"ALERT: Evaluation failed with score {score.score}")
|
||||
print(f"Reasoning: {score.reasoning}")
|
||||
|
||||
callback_evaluator = CriteriaEvaluator(
|
||||
criteria="Response is professional and helpful",
|
||||
agent=agent,
|
||||
input_text="Help me",
|
||||
on_fail=handle_failure,
|
||||
threshold=8.0
|
||||
)
|
||||
|
||||
callback_evaluator.run()
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Skipping agent-based criteria evaluation...")
|
||||
|
||||
# Test pre-generated output evaluation (doesn't need agent)
|
||||
print("\n--- Testing Pre-generated Output Evaluation ---")
|
||||
if has_api_key:
|
||||
output = "I understand your frustration. Let me check on your order right away."
|
||||
try:
|
||||
result2 = evaluator.evaluate_output(output, print_summary=True)
|
||||
print(f"Pre-generated output score: {result2.avg_score}/10")
|
||||
except Exception as e:
|
||||
print(f"Pre-generated evaluation failed: {e}")
|
||||
else:
|
||||
print("⚠️ Skipping pre-generated evaluation (no API key)")
|
||||
print("To run full evaluation, set OPENAI_API_KEY environment variable")
|
||||
@@ -0,0 +1,59 @@
|
||||
"""
|
||||
Performance Evaluation Example
|
||||
|
||||
This example demonstrates how to benchmark agent performance
|
||||
by measuring runtime and memory usage across multiple iterations.
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.eval import PerformanceEvaluator
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
if has_api_key:
|
||||
print("--- Testing Agent Performance ---")
|
||||
# Create a simple agent
|
||||
agent = Agent(
|
||||
instructions="You are a helpful assistant. Answer questions briefly."
|
||||
)
|
||||
|
||||
# Create performance evaluator
|
||||
evaluator = PerformanceEvaluator(
|
||||
agent=agent,
|
||||
input_text="What is the capital of France?",
|
||||
num_iterations=10, # Run 10 benchmark iterations
|
||||
warmup_runs=2, # 2 warmup runs before measurement
|
||||
track_memory=True, # Track memory usage
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Run evaluation
|
||||
result = evaluator.run(print_summary=True)
|
||||
|
||||
# Access detailed metrics
|
||||
print("\nAgent Performance Results:")
|
||||
print(f" Average Time: {result.avg_run_time:.4f}s")
|
||||
print(f" Min Time: {result.min_run_time:.4f}s")
|
||||
print(f" Max Time: {result.max_run_time:.4f}s")
|
||||
print(f" Median Time: {result.median_run_time:.4f}s")
|
||||
print(f" P95 Time: {result.p95_run_time:.4f}s")
|
||||
print(f" Avg Memory: {result.avg_memory:.2f} MB")
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Skipping agent performance test...")
|
||||
|
||||
# You can also benchmark any function
|
||||
def my_function():
|
||||
import time
|
||||
time.sleep(0.1)
|
||||
return "done"
|
||||
|
||||
func_evaluator = PerformanceEvaluator(
|
||||
func=my_function,
|
||||
num_iterations=5,
|
||||
warmup_runs=1,
|
||||
track_memory=False
|
||||
)
|
||||
|
||||
func_result = func_evaluator.run(print_summary=True)
|
||||
@@ -0,0 +1,72 @@
|
||||
"""
|
||||
Reliability Evaluation Example
|
||||
|
||||
This example demonstrates how to verify that agents call
|
||||
the expected tools during execution.
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.eval import ReliabilityEvaluator
|
||||
|
||||
|
||||
def search_web(query: str) -> str:
|
||||
"""Search the web for information."""
|
||||
return f"Search results for: {query}"
|
||||
|
||||
|
||||
def calculate(expression: str) -> str:
|
||||
"""Calculate a math expression."""
|
||||
return str(eval(expression))
|
||||
|
||||
|
||||
# Check if we have an API key
|
||||
has_api_key = os.getenv("OPENAI_API_KEY") is not None
|
||||
|
||||
if has_api_key:
|
||||
print("--- Testing Agent Tool Call Reliability ---")
|
||||
# Create an agent with tools
|
||||
agent = Agent(
|
||||
instructions="You are a helpful assistant with access to search and calculator tools.",
|
||||
tools=[search_web, calculate]
|
||||
)
|
||||
|
||||
# Create reliability evaluator
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="Search for the weather in Paris and calculate 25 * 4",
|
||||
expected_tools=["search_web", "calculate"], # Tools that should be called
|
||||
forbidden_tools=["delete_file"], # Tools that should NOT be called
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Run evaluation
|
||||
result = evaluator.run(print_summary=True)
|
||||
|
||||
# Check results
|
||||
print("\nAgent Reliability Results:")
|
||||
print(f" Passed: {result.passed}")
|
||||
print(f" Pass Rate: {result.pass_rate:.1%}")
|
||||
else:
|
||||
print("⚠️ No OPENAI_API_KEY found. Skipping agent reliability test...")
|
||||
print("Agent would fail to call tools without API key (expected behavior)")
|
||||
|
||||
# You can also evaluate pre-recorded tool calls (doesn't need API key)
|
||||
print("\n--- Testing Pre-recorded Tool Call Evaluation ---")
|
||||
# Create a mock agent for tool call evaluation
|
||||
mock_agent = Agent(instructions="Mock agent")
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=mock_agent,
|
||||
expected_tools=["search_web", "calculate"], # Tools that should be called
|
||||
forbidden_tools=["delete_file"], # Tools that should NOT be called
|
||||
verbose=True
|
||||
)
|
||||
|
||||
result2 = evaluator.evaluate_tool_calls(
|
||||
actual_tools=["search_web", "calculate"],
|
||||
print_summary=True
|
||||
)
|
||||
|
||||
print("\nPre-recorded Tool Call Results:")
|
||||
print(f" Passed: {result2.passed}")
|
||||
print(f" Pass Rate: {result2.pass_rate:.1%}")
|
||||
@@ -0,0 +1,115 @@
|
||||
"""
|
||||
24/7 News Checker Agent Example
|
||||
|
||||
This example demonstrates how to create a continuously running agent
|
||||
that checks for AI news every hour and prints the results.
|
||||
|
||||
Usage:
|
||||
python news_checker_agent.py
|
||||
|
||||
The agent will:
|
||||
1. Run immediately on start
|
||||
2. Check for latest AI news every 1 hour
|
||||
3. Print results to console
|
||||
4. Continue running until stopped (Ctrl+C)
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
# Add parent directory to path for imports
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../..'))
|
||||
|
||||
from praisonaiagents import Agent
|
||||
from praisonai.agent_scheduler import AgentScheduler
|
||||
from tools import search_tool
|
||||
|
||||
|
||||
def on_success(result):
|
||||
"""Callback function called when agent execution succeeds."""
|
||||
print("\n" + "="*80)
|
||||
print(f"✅ SUCCESS - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print("="*80)
|
||||
print(result)
|
||||
print("="*80 + "\n")
|
||||
|
||||
|
||||
def on_failure(error):
|
||||
"""Callback function called when agent execution fails."""
|
||||
print("\n" + "="*80)
|
||||
print(f"❌ FAILURE - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print("="*80)
|
||||
print(f"Error: {error}")
|
||||
print("="*80 + "\n")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function to set up and run the scheduled news checker agent."""
|
||||
|
||||
print("\n🤖 Starting 24/7 News Checker Agent")
|
||||
print("="*80)
|
||||
|
||||
# Create the agent with web search capability
|
||||
agent = Agent(
|
||||
name="NewsChecker",
|
||||
role="AI News Monitor",
|
||||
goal="Monitor and summarize the latest AI and technology news",
|
||||
backstory="""You are an expert AI news analyst who stays up-to-date
|
||||
with the latest developments in artificial intelligence, machine learning,
|
||||
and technology. You provide concise, accurate summaries of important news.""",
|
||||
tools=[search_tool],
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Define the task
|
||||
task = """Search for the latest AI news from today.
|
||||
Focus on major developments, breakthroughs, or announcements.
|
||||
Provide a summary of the top 3-5 most important news items.
|
||||
Include the source and key details for each item."""
|
||||
|
||||
# Create scheduler with callbacks
|
||||
scheduler = AgentScheduler(
|
||||
agent=agent,
|
||||
task=task,
|
||||
on_success=on_success,
|
||||
on_failure=on_failure
|
||||
)
|
||||
|
||||
print(f"Agent: {agent.name}")
|
||||
print(f"Task: {task}")
|
||||
print(f"Schedule: Every 1 hour")
|
||||
print("="*80)
|
||||
print("\n⏰ Starting scheduler... (Press Ctrl+C to stop)\n")
|
||||
|
||||
# Start the scheduler
|
||||
# - "hourly" or "*/1h" = every 1 hour
|
||||
# - run_immediately=True = run once before starting the schedule
|
||||
scheduler.start(
|
||||
schedule_expr="hourly", # Run every hour
|
||||
max_retries=3, # Retry up to 3 times on failure
|
||||
run_immediately=True # Run immediately on start
|
||||
)
|
||||
|
||||
try:
|
||||
# Keep the main thread alive
|
||||
while scheduler.is_running:
|
||||
import time
|
||||
time.sleep(1)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n🛑 Stopping scheduler...")
|
||||
scheduler.stop()
|
||||
|
||||
# Print final statistics
|
||||
stats = scheduler.get_stats()
|
||||
print("\n📊 Final Statistics:")
|
||||
print(f" Total Executions: {stats['total_executions']}")
|
||||
print(f" Successful: {stats['successful_executions']}")
|
||||
print(f" Failed: {stats['failed_executions']}")
|
||||
print(f" Success Rate: {stats['success_rate']:.1f}%")
|
||||
print("\n✅ Agent stopped successfully\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,143 @@
|
||||
"""
|
||||
Live News Checker Agent - 24/7 Scheduled Agent
|
||||
|
||||
This example demonstrates a real working agent that checks AI news
|
||||
every hour using the AgentScheduler with actual API calls.
|
||||
|
||||
Requirements:
|
||||
- OPENAI_API_KEY environment variable set
|
||||
- praisonaiagents package installed
|
||||
|
||||
Usage:
|
||||
python news_checker_live.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
# Add parent directory to path
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../..'))
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../../..'))
|
||||
|
||||
from praisonaiagents import Agent
|
||||
from praisonai.scheduler import AgentScheduler
|
||||
|
||||
# Import search_tool from root tools.py
|
||||
try:
|
||||
from tools import search_tool
|
||||
except ImportError:
|
||||
# Fallback: create inline search tool
|
||||
from duckduckgo_search import DDGS
|
||||
|
||||
def search_tool(query: str) -> list:
|
||||
"""Search the web using DuckDuckGo."""
|
||||
try:
|
||||
results = []
|
||||
ddgs = DDGS()
|
||||
for result in ddgs.text(keywords=query, max_results=5):
|
||||
results.append({
|
||||
"title": result.get("title", ""),
|
||||
"url": result.get("href", ""),
|
||||
"snippet": result.get("body", ""),
|
||||
})
|
||||
return results
|
||||
except Exception as e:
|
||||
print(f"Search error: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def on_success(result):
|
||||
"""Callback when agent execution succeeds."""
|
||||
print("\n" + "="*80)
|
||||
print(f"✅ SUCCESS - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print("="*80)
|
||||
print(result)
|
||||
print("="*80 + "\n")
|
||||
|
||||
|
||||
def on_failure(error):
|
||||
"""Callback when agent execution fails."""
|
||||
print("\n" + "="*80)
|
||||
print(f"❌ FAILURE - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print("="*80)
|
||||
print(f"Error: {error}")
|
||||
print("="*80 + "\n")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function to run the scheduled news checker."""
|
||||
|
||||
# Check for API key
|
||||
if not os.getenv("OPENAI_API_KEY"):
|
||||
print("❌ Error: OPENAI_API_KEY environment variable not set")
|
||||
print("Please set it with: export OPENAI_API_KEY=your_key_here")
|
||||
sys.exit(1)
|
||||
|
||||
print("\n🤖 Starting 24/7 AI News Checker Agent")
|
||||
print("="*80)
|
||||
|
||||
# Create the agent with web search capability
|
||||
agent = Agent(
|
||||
name="AI News Monitor",
|
||||
role="Technology News Analyst",
|
||||
goal="Monitor and summarize the latest AI and technology news",
|
||||
backstory="""You are an expert AI news analyst who stays up-to-date
|
||||
with the latest developments in artificial intelligence, machine learning,
|
||||
and technology. You provide concise, accurate summaries of important news.""",
|
||||
tools=[search_tool],
|
||||
verbose=True,
|
||||
self_reflect=False # Disable reflection for faster execution
|
||||
)
|
||||
|
||||
# Define the task
|
||||
task = """Search for the latest AI news from today.
|
||||
Focus on major developments, breakthroughs, or announcements.
|
||||
Provide a summary of the top 3 most important news items.
|
||||
Include the source and key details for each item."""
|
||||
|
||||
# Create scheduler with callbacks
|
||||
scheduler = AgentScheduler(
|
||||
agent=agent,
|
||||
task=task,
|
||||
on_success=on_success,
|
||||
on_failure=on_failure
|
||||
)
|
||||
|
||||
print(f"Agent: {agent.name}")
|
||||
print(f"Task: {task}")
|
||||
print(f"Schedule: Every 1 hour")
|
||||
print("="*80)
|
||||
print("\n⏰ Starting scheduler... (Press Ctrl+C to stop)\n")
|
||||
|
||||
# Start the scheduler
|
||||
# For testing: use "*/2m" (every 2 minutes) instead of "hourly"
|
||||
# For production: use "hourly"
|
||||
scheduler.start(
|
||||
schedule_expr="*/2m", # Every 2 minutes for testing
|
||||
max_retries=3,
|
||||
run_immediately=True # Run once immediately
|
||||
)
|
||||
|
||||
try:
|
||||
# Keep the main thread alive
|
||||
while scheduler.is_running:
|
||||
import time
|
||||
time.sleep(1)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n🛑 Stopping scheduler...")
|
||||
scheduler.stop()
|
||||
|
||||
# Print final statistics
|
||||
stats = scheduler.get_stats()
|
||||
print("\n📊 Final Statistics:")
|
||||
print(f" Total Executions: {stats['total_executions']}")
|
||||
print(f" Successful: {stats['successful_executions']}")
|
||||
print(f" Failed: {stats['failed_executions']}")
|
||||
print(f" Success Rate: {stats['success_rate']:.1f}%")
|
||||
print("\n✅ Agent stopped successfully\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,61 @@
|
||||
"""
|
||||
Simple test script for 24/7 Agent Scheduler
|
||||
|
||||
This script demonstrates the basic usage of AgentScheduler
|
||||
for running agents periodically.
|
||||
|
||||
Usage:
|
||||
python simple_test.py
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../..'))
|
||||
|
||||
from unittest.mock import Mock
|
||||
from praisonai.scheduler import AgentScheduler
|
||||
|
||||
# Create a mock agent for testing
|
||||
mock_agent = Mock()
|
||||
mock_agent.name = "TestAgent"
|
||||
mock_agent.start = Mock(return_value="Test execution successful!")
|
||||
|
||||
# Create scheduler
|
||||
scheduler = AgentScheduler(
|
||||
agent=mock_agent,
|
||||
task="Test task execution"
|
||||
)
|
||||
|
||||
print("🤖 Testing AgentScheduler")
|
||||
print("=" * 60)
|
||||
|
||||
# Test 1: Execute once
|
||||
print("\n✅ Test 1: Execute once")
|
||||
result = scheduler.execute_once()
|
||||
print(f"Result: {result}")
|
||||
|
||||
# Test 2: Get stats
|
||||
print("\n✅ Test 2: Get statistics")
|
||||
stats = scheduler.get_stats()
|
||||
print(f"Stats: {stats}")
|
||||
|
||||
# Test 3: Start scheduler (will run every 5 seconds)
|
||||
print("\n✅ Test 3: Start scheduler (*/5s)")
|
||||
scheduler.start("*/5s", run_immediately=True)
|
||||
|
||||
import time
|
||||
print("Running for 12 seconds...")
|
||||
time.sleep(12)
|
||||
|
||||
# Stop scheduler
|
||||
print("\n✅ Test 4: Stop scheduler")
|
||||
scheduler.stop()
|
||||
|
||||
# Final stats
|
||||
print("\n📊 Final Statistics:")
|
||||
final_stats = scheduler.get_stats()
|
||||
for key, value in final_stats.items():
|
||||
print(f" {key}: {value}")
|
||||
|
||||
print("\n✅ All tests passed!")
|
||||
print("=" * 60)
|
||||
@@ -0,0 +1,14 @@
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "5 Second Timer"
|
||||
role: "Timer Agent"
|
||||
instructions: "Display the current time"
|
||||
verbose: true
|
||||
|
||||
task: "Show current time"
|
||||
|
||||
schedule:
|
||||
interval: "*/5s"
|
||||
max_retries: 3
|
||||
run_immediately: true
|
||||
@@ -0,0 +1,14 @@
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "Counter Agent"
|
||||
role: "Counter"
|
||||
instructions: "Count from 1 to 5, incrementing by 1 each time. Just output the numbers clearly."
|
||||
verbose: true
|
||||
|
||||
task: "Print numbers 1, 2, 3, 4, 5 (one number per line)"
|
||||
|
||||
schedule:
|
||||
interval: "*/5s"
|
||||
max_retries: 3
|
||||
run_immediately: true
|
||||
@@ -0,0 +1,16 @@
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "Counter Agent with Limits"
|
||||
role: "Counter"
|
||||
instructions: "Count from 1 to 5, incrementing by 1 each time. Just output the numbers clearly."
|
||||
verbose: true
|
||||
|
||||
task: "Print numbers 1, 2, 3, 4, 5 (one number per line)"
|
||||
|
||||
schedule:
|
||||
interval: "*/5s" # Run every 5 seconds
|
||||
max_retries: 3 # Retry up to 3 times on failure
|
||||
run_immediately: true # Run once immediately on start
|
||||
timeout: 30 # Max 30 seconds per execution
|
||||
max_cost: 0.01 # Stop after spending $0.01 USD
|
||||
@@ -0,0 +1,15 @@
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "Budget Test Agent"
|
||||
role: "Tester"
|
||||
instructions: "Count from 1 to 3"
|
||||
verbose: true
|
||||
|
||||
task: "Print numbers 1, 2, 3"
|
||||
|
||||
schedule:
|
||||
interval: "*/3s"
|
||||
max_retries: 2
|
||||
run_immediately: true
|
||||
# No max_cost specified - should default to $1.00
|
||||
@@ -0,0 +1,73 @@
|
||||
"""
|
||||
Basic Agent Skills Usage Example
|
||||
|
||||
This example demonstrates how to use Agent Skills with PraisonAI Agents.
|
||||
Skills are modular capabilities that can be loaded and used by agents.
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent, SkillManager
|
||||
|
||||
# Example 1: Using SkillManager directly
|
||||
print("=" * 50)
|
||||
print("Example 1: Using SkillManager directly")
|
||||
print("=" * 50)
|
||||
|
||||
# Create a skill manager
|
||||
manager = SkillManager()
|
||||
|
||||
# Discover skills from a directory
|
||||
# This scans the directory for subdirectories containing SKILL.md files
|
||||
skill_count = manager.discover(["./pdf-processing"], include_defaults=False)
|
||||
print(f"Discovered {skill_count} skill(s)")
|
||||
|
||||
# List available skills
|
||||
for skill in manager:
|
||||
print(f" - {skill.properties.name}: {skill.properties.description[:50]}...")
|
||||
|
||||
# Generate prompt XML for system prompt injection
|
||||
prompt_xml = manager.to_prompt()
|
||||
print("\nGenerated prompt XML:")
|
||||
print(prompt_xml)
|
||||
|
||||
# Example 2: Using skills with an Agent
|
||||
print("\n" + "=" * 50)
|
||||
print("Example 2: Using skills with an Agent")
|
||||
print("=" * 50)
|
||||
|
||||
# Create an agent with skills
|
||||
# Skills are lazy-loaded only when accessed
|
||||
agent = Agent(
|
||||
name="PDF Assistant",
|
||||
instructions="You are a helpful assistant that can process PDF documents.",
|
||||
skills=["./pdf-processing"], # Direct skill paths
|
||||
)
|
||||
|
||||
# Access the skill manager through the agent
|
||||
if agent.skill_manager:
|
||||
print(f"Agent has {len(agent.skill_manager)} skill(s) loaded")
|
||||
|
||||
# Get the skills prompt for system prompt injection
|
||||
skills_prompt = agent.get_skills_prompt()
|
||||
print("\nAgent skills prompt:")
|
||||
print(skills_prompt)
|
||||
|
||||
# Example 3: Discovering skills from multiple directories
|
||||
print("\n" + "=" * 50)
|
||||
print("Example 3: Using skills_dirs for discovery")
|
||||
print("=" * 50)
|
||||
|
||||
# Create an agent that discovers skills from directories
|
||||
agent_with_discovery = Agent(
|
||||
name="Multi-Skill Agent",
|
||||
instructions="You are a versatile assistant with multiple skills.",
|
||||
skills_dirs=["./"], # Scan current directory for skill subdirectories
|
||||
)
|
||||
|
||||
if agent_with_discovery.skill_manager:
|
||||
print(f"Discovered {len(agent_with_discovery.skill_manager)} skill(s)")
|
||||
for name in agent_with_discovery.skill_manager.skill_names:
|
||||
print(f" - {name}")
|
||||
|
||||
print("\n" + "=" * 50)
|
||||
print("Examples complete!")
|
||||
print("=" * 50)
|
||||
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Example: Creating Skills with PraisonAI CLI
|
||||
|
||||
This example demonstrates how to create Agent Skills using the CLI.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import tempfile
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def example_create_skill_with_template():
|
||||
"""Create a skill using template mode (no API key required)."""
|
||||
print("=" * 60)
|
||||
print("Example 1: Create skill with template")
|
||||
print("=" * 60)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
result = subprocess.run([
|
||||
"praisonai", "skills", "create",
|
||||
"--name", "my-template-skill",
|
||||
"--description", "A skill for processing text files",
|
||||
"--author", "example-team",
|
||||
"--license", "MIT",
|
||||
"--template",
|
||||
"--output-dir", tmpdir
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
if result.returncode == 0:
|
||||
skill_md = Path(tmpdir) / "my-template-skill" / "SKILL.md"
|
||||
print(f"\nGenerated SKILL.md:\n{skill_md.read_text()[:500]}...")
|
||||
else:
|
||||
print(f"Error: {result.stderr}")
|
||||
|
||||
|
||||
def example_create_skill_with_ai():
|
||||
"""Create a skill using AI generation (requires API key)."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 2: Create skill with AI generation")
|
||||
print("=" * 60)
|
||||
|
||||
if not os.environ.get("OPENAI_API_KEY"):
|
||||
print("Skipping: OPENAI_API_KEY not set")
|
||||
return
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
result = subprocess.run([
|
||||
"praisonai", "skills", "create",
|
||||
"--name", "csv-analyzer",
|
||||
"--description", "Analyze CSV files and generate statistical reports",
|
||||
"--output-dir", tmpdir
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
if result.returncode == 0:
|
||||
skill_md = Path(tmpdir) / "csv-analyzer" / "SKILL.md"
|
||||
print(f"\nGenerated SKILL.md:\n{skill_md.read_text()[:800]}...")
|
||||
|
||||
# Check if script was generated
|
||||
script_path = Path(tmpdir) / "csv-analyzer" / "scripts" / "skill.py"
|
||||
if script_path.exists():
|
||||
print(f"\nGenerated skill.py:\n{script_path.read_text()[:500]}...")
|
||||
|
||||
|
||||
def example_create_skill_with_script():
|
||||
"""Create a skill with auto-generated Python script."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 3: Create skill with script")
|
||||
print("=" * 60)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
result = subprocess.run([
|
||||
"praisonai", "skills", "create",
|
||||
"--name", "data-processor",
|
||||
"--description", "Process and transform data files",
|
||||
"--template",
|
||||
"--script",
|
||||
"--output-dir", tmpdir
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
if result.returncode == 0:
|
||||
script_path = Path(tmpdir) / "data-processor" / "scripts" / "skill.py"
|
||||
if script_path.exists():
|
||||
print(f"\nGenerated skill.py:\n{script_path.read_text()}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
example_create_skill_with_template()
|
||||
example_create_skill_with_ai()
|
||||
example_create_skill_with_script()
|
||||
@@ -0,0 +1,163 @@
|
||||
"""
|
||||
Custom Skill Creation Example
|
||||
|
||||
This example demonstrates how to programmatically create and validate
|
||||
custom Agent Skills using PraisonAI Agents.
|
||||
"""
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from praisonaiagents.skills import (
|
||||
SkillManager,
|
||||
SkillLoader,
|
||||
validate,
|
||||
read_properties,
|
||||
)
|
||||
|
||||
|
||||
def create_custom_skill(
|
||||
name: str,
|
||||
description: str,
|
||||
instructions: str,
|
||||
base_dir: str = None
|
||||
) -> Path:
|
||||
"""Create a custom skill programmatically.
|
||||
|
||||
Args:
|
||||
name: Skill name (kebab-case)
|
||||
description: What the skill does
|
||||
instructions: Markdown instructions for the skill
|
||||
base_dir: Directory to create skill in (default: temp dir)
|
||||
|
||||
Returns:
|
||||
Path to the created skill directory
|
||||
"""
|
||||
if base_dir is None:
|
||||
base_dir = tempfile.mkdtemp()
|
||||
|
||||
skill_dir = Path(base_dir) / name
|
||||
skill_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create SKILL.md
|
||||
skill_md_content = f"""---
|
||||
name: {name}
|
||||
description: {description}
|
||||
license: Apache-2.0
|
||||
metadata:
|
||||
author: custom
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
{instructions}
|
||||
"""
|
||||
(skill_dir / "SKILL.md").write_text(skill_md_content)
|
||||
|
||||
# Create optional directories
|
||||
(skill_dir / "scripts").mkdir(exist_ok=True)
|
||||
(skill_dir / "references").mkdir(exist_ok=True)
|
||||
(skill_dir / "assets").mkdir(exist_ok=True)
|
||||
|
||||
return skill_dir
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 60)
|
||||
print("Custom Skill Creation Example")
|
||||
print("=" * 60)
|
||||
|
||||
# Create a custom skill
|
||||
print("\n1. Creating a custom skill...")
|
||||
|
||||
skill_path = create_custom_skill(
|
||||
name="code-review",
|
||||
description="Review code for best practices, bugs, and improvements. Use when asked to review or analyze code quality.",
|
||||
instructions="""
|
||||
# Code Review Skill
|
||||
|
||||
## Overview
|
||||
This skill enables thorough code review with focus on:
|
||||
- Code quality and readability
|
||||
- Potential bugs and edge cases
|
||||
- Performance considerations
|
||||
- Security vulnerabilities
|
||||
- Best practices adherence
|
||||
|
||||
## Instructions
|
||||
|
||||
When reviewing code:
|
||||
|
||||
1. **Read the code carefully** - Understand the purpose and flow
|
||||
2. **Check for bugs** - Look for null checks, edge cases, error handling
|
||||
3. **Evaluate readability** - Variable names, comments, structure
|
||||
4. **Consider performance** - Time/space complexity, unnecessary operations
|
||||
5. **Security review** - Input validation, injection risks, data exposure
|
||||
6. **Suggest improvements** - Provide actionable recommendations
|
||||
|
||||
## Output Format
|
||||
|
||||
Structure your review as:
|
||||
- Summary: Brief overview of the code
|
||||
- Issues: List of problems found (severity: high/medium/low)
|
||||
- Suggestions: Recommended improvements
|
||||
- Positive aspects: What's done well
|
||||
"""
|
||||
)
|
||||
|
||||
print(f" Created skill at: {skill_path}")
|
||||
|
||||
# Validate the skill
|
||||
print("\n2. Validating the skill...")
|
||||
errors = validate(skill_path)
|
||||
|
||||
if errors:
|
||||
print(" Validation errors:")
|
||||
for error in errors:
|
||||
print(f" - {error}")
|
||||
else:
|
||||
print(" ✓ Skill is valid!")
|
||||
|
||||
# Load and inspect the skill
|
||||
print("\n3. Loading skill properties...")
|
||||
props = read_properties(skill_path)
|
||||
print(f" Name: {props.name}")
|
||||
print(f" Description: {props.description[:60]}...")
|
||||
print(f" License: {props.license}")
|
||||
|
||||
# Use with SkillLoader for progressive loading
|
||||
print("\n4. Using SkillLoader for progressive disclosure...")
|
||||
loader = SkillLoader()
|
||||
|
||||
# Level 1: Load metadata only
|
||||
skill = loader.load_metadata(str(skill_path))
|
||||
print(f" Level 1 - Metadata loaded: {skill.properties.name}")
|
||||
print(f" Instructions loaded: {skill.is_activated}")
|
||||
|
||||
# Level 2: Activate (load instructions)
|
||||
loader.activate(skill)
|
||||
print(f" Level 2 - Activated: {skill.is_activated}")
|
||||
print(f" Instructions preview: {skill.instructions[:100]}...")
|
||||
|
||||
# Use with SkillManager
|
||||
print("\n5. Using with SkillManager...")
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_path))
|
||||
|
||||
# Generate prompt XML
|
||||
prompt = manager.to_prompt()
|
||||
print(" Generated prompt XML:")
|
||||
print(prompt)
|
||||
|
||||
# Clean up
|
||||
print("\n6. Cleanup...")
|
||||
import shutil
|
||||
shutil.rmtree(skill_path.parent)
|
||||
print(" Temporary files removed.")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Example complete!")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Example: Multi-Agent CLI Usage
|
||||
|
||||
This example demonstrates how to use the praisonai agents command
|
||||
to define and run multiple agents from the command line.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import os
|
||||
|
||||
|
||||
def example_single_agent():
|
||||
"""Run a single agent with a simple task."""
|
||||
print("=" * 60)
|
||||
print("Example 1: Single Agent")
|
||||
print("=" * 60)
|
||||
|
||||
result = subprocess.run([
|
||||
"praisonai", "agents", "run",
|
||||
"--agent", "assistant:Helper",
|
||||
"--task", "What is the capital of France?"
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
if result.returncode != 0:
|
||||
print(f"Error: {result.stderr}")
|
||||
|
||||
|
||||
def example_agent_with_tools():
|
||||
"""Run an agent with tools."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 2: Agent with Tools")
|
||||
print("=" * 60)
|
||||
|
||||
result = subprocess.run([
|
||||
"praisonai", "agents", "run",
|
||||
"--agent", "researcher:Research Analyst:internet_search",
|
||||
"--task", "Find the latest news about renewable energy"
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
|
||||
|
||||
def example_multiple_agents_sequential():
|
||||
"""Run multiple agents in sequential mode."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 3: Multiple Agents (Sequential)")
|
||||
print("=" * 60)
|
||||
|
||||
result = subprocess.run([
|
||||
"praisonai", "agents", "run",
|
||||
"--agent", "researcher:Research Analyst:internet_search",
|
||||
"--agent", "writer:Content Writer:write_file",
|
||||
"--task", "Research AI trends and write a brief summary"
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
|
||||
|
||||
def example_multiple_agents_parallel():
|
||||
"""Run multiple agents in parallel mode."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 4: Multiple Agents (Parallel)")
|
||||
print("=" * 60)
|
||||
|
||||
result = subprocess.run([
|
||||
"praisonai", "agents", "run",
|
||||
"--agent", "analyst1:Market Analyst",
|
||||
"--agent", "analyst2:Tech Analyst",
|
||||
"--process", "parallel",
|
||||
"--task", "Analyze the current state of AI industry"
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
|
||||
|
||||
def example_with_custom_llm():
|
||||
"""Run agents with a custom LLM model."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 5: Custom LLM Model")
|
||||
print("=" * 60)
|
||||
|
||||
result = subprocess.run([
|
||||
"praisonai", "agents", "run",
|
||||
"--agent", "coder:Developer:execute_command",
|
||||
"--llm", "gpt-4o-mini",
|
||||
"--task", "Write a simple hello world Python script"
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
|
||||
|
||||
def example_with_instructions():
|
||||
"""Run agents with additional instructions."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 6: With Additional Instructions")
|
||||
print("=" * 60)
|
||||
|
||||
result = subprocess.run([
|
||||
"praisonai", "agents", "run",
|
||||
"--agent", "analyst:Data Analyst",
|
||||
"--instructions", "Be concise and provide only key insights",
|
||||
"--task", "Explain the benefits of using AI in healthcare"
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
|
||||
|
||||
def example_list_tools():
|
||||
"""List available tools for agents."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 7: List Available Tools")
|
||||
print("=" * 60)
|
||||
|
||||
result = subprocess.run([
|
||||
"praisonai", "agents", "list"
|
||||
], capture_output=True, text=True)
|
||||
|
||||
print(result.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Check for API key
|
||||
if not os.environ.get("OPENAI_API_KEY"):
|
||||
print("Warning: OPENAI_API_KEY not set. Some examples may fail.")
|
||||
print()
|
||||
|
||||
# Run examples
|
||||
example_list_tools()
|
||||
example_single_agent()
|
||||
# Uncomment to run more examples:
|
||||
# example_agent_with_tools()
|
||||
# example_multiple_agents_sequential()
|
||||
# example_multiple_agents_parallel()
|
||||
# example_with_custom_llm()
|
||||
# example_with_instructions()
|
||||
@@ -0,0 +1,38 @@
|
||||
---
|
||||
name: pdf-processing
|
||||
description: Process and extract information from PDF documents. Use this skill when the user asks to read, analyze, or extract data from PDF files.
|
||||
license: Apache-2.0
|
||||
compatibility: Works with PraisonAI Agents
|
||||
metadata:
|
||||
author: praisonai
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
# PDF Processing Skill
|
||||
|
||||
## Overview
|
||||
|
||||
This skill enables agents to process PDF documents, extract text content, and analyze the information within them.
|
||||
|
||||
## When to Use
|
||||
|
||||
Activate this skill when:
|
||||
- User asks to read or analyze a PDF file
|
||||
- User needs to extract text from a PDF
|
||||
- User wants to summarize a PDF document
|
||||
- User needs to search for information within a PDF
|
||||
|
||||
## Instructions
|
||||
|
||||
1. First, verify the PDF file exists at the specified path
|
||||
2. Use appropriate tools to read the PDF content
|
||||
3. Extract text while preserving structure where possible
|
||||
4. For large PDFs, process in chunks to manage context
|
||||
5. Summarize or analyze based on user's specific request
|
||||
|
||||
## Best Practices
|
||||
|
||||
- Always confirm the file path before processing
|
||||
- Handle encrypted PDFs gracefully with appropriate error messages
|
||||
- For scanned PDFs, note that OCR may be required
|
||||
- Preserve important formatting like tables and lists
|
||||
@@ -3,7 +3,7 @@ Praison AI Agents - A package for hierarchical AI agent task execution
|
||||
"""
|
||||
|
||||
# Apply warning patch BEFORE any imports to intercept warnings at the source
|
||||
from . import _warning_patch
|
||||
from . import _warning_patch # noqa: F401
|
||||
|
||||
# Import centralized logging configuration FIRST
|
||||
from . import _logging
|
||||
@@ -78,6 +78,19 @@ def __getattr__(name):
|
||||
elif name == "LineRange":
|
||||
from praisonaiagents.context.fast import LineRange
|
||||
return LineRange
|
||||
# Agent Skills support (lazy loaded for zero performance impact)
|
||||
elif name == "SkillManager":
|
||||
from praisonaiagents.skills import SkillManager
|
||||
return SkillManager
|
||||
elif name == "SkillProperties":
|
||||
from praisonaiagents.skills import SkillProperties
|
||||
return SkillProperties
|
||||
elif name == "SkillMetadata":
|
||||
from praisonaiagents.skills import SkillMetadata
|
||||
return SkillMetadata
|
||||
elif name == "SkillLoader":
|
||||
from praisonaiagents.skills import SkillLoader
|
||||
return SkillLoader
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
# Planning mode support
|
||||
@@ -289,6 +302,23 @@ __all__ = [
|
||||
'ExpandResult'
|
||||
]
|
||||
|
||||
# Add workflow exports
|
||||
__all__.extend([
|
||||
'Workflow',
|
||||
'WorkflowStep',
|
||||
'WorkflowContext',
|
||||
'StepResult',
|
||||
'Route',
|
||||
'Parallel',
|
||||
'Loop',
|
||||
'Repeat',
|
||||
'route',
|
||||
'parallel',
|
||||
'loop',
|
||||
'repeat',
|
||||
'Pipeline'
|
||||
])
|
||||
|
||||
# Add MCP to __all__ if available
|
||||
if _mcp_available:
|
||||
__all__.append('MCP')
|
||||
@@ -305,6 +335,14 @@ __all__.extend([
|
||||
'LineRange'
|
||||
])
|
||||
|
||||
# Add Agent Skills exports (lazy loaded)
|
||||
__all__.extend([
|
||||
'SkillManager',
|
||||
'SkillProperties',
|
||||
'SkillMetadata',
|
||||
'SkillLoader'
|
||||
])
|
||||
|
||||
# AG-UI support (optional, lazy loaded)
|
||||
try:
|
||||
from praisonaiagents.ui.agui import AGUI
|
||||
|
||||
@@ -55,10 +55,10 @@ def _get_all_noisy_loggers() -> List[str]:
|
||||
|
||||
def _configure_loggers():
|
||||
"""Configure all loggers based on LOGLEVEL environment variable."""
|
||||
loglevel = os.environ.get('LOGLEVEL', 'INFO').upper()
|
||||
loglevel = os.environ.get('LOGLEVEL', 'WARNING').upper()
|
||||
|
||||
# When DEBUG is set, allow some HTTP logging for API endpoints
|
||||
if loglevel == 'DEBUG':
|
||||
# When DEBUG or INFO is set, allow some HTTP logging for API endpoints
|
||||
if loglevel in ('DEBUG', 'INFO'):
|
||||
allowed_debug_loggers = {"httpx", "httpx._client", "openai._client"}
|
||||
|
||||
for logger_name in _get_all_noisy_loggers():
|
||||
@@ -109,10 +109,10 @@ def configure_root_logger():
|
||||
"""Configure the root logger with RichHandler."""
|
||||
from rich.logging import RichHandler
|
||||
|
||||
loglevel = os.environ.get('LOGLEVEL', 'INFO').upper()
|
||||
loglevel = os.environ.get('LOGLEVEL', 'WARNING').upper()
|
||||
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, loglevel, logging.INFO),
|
||||
level=getattr(logging, loglevel, logging.WARNING),
|
||||
format="%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s",
|
||||
datefmt="[%X]",
|
||||
handlers=[RichHandler(rich_tracebacks=True)],
|
||||
|
||||
@@ -8,9 +8,10 @@ import functools
|
||||
import sys
|
||||
|
||||
# Apply aggressive warning filters first
|
||||
warnings.filterwarnings("ignore", message=".*There is no current event loop.*", category=DeprecationWarning)
|
||||
warnings.filterwarnings("ignore", message=".*Pydantic serializer warnings.*")
|
||||
warnings.filterwarnings("ignore", message=".*PydanticSerializationUnexpectedValue.*")
|
||||
warnings.filterwarnings("ignore", message=".*Expected 9 fields but got.*")
|
||||
warnings.filterwarnings("ignore", message=".*Expected \\d+ fields but got.*")
|
||||
warnings.filterwarnings("ignore", message=".*Expected `StreamingChoices` but got.*")
|
||||
warnings.filterwarnings("ignore", message=".*serialized value may not be as expected.*")
|
||||
warnings.filterwarnings("ignore", message=".*Use 'content=<...>' to upload raw bytes/text content.*")
|
||||
@@ -27,11 +28,12 @@ SUPPRESSED_PATTERNS = [
|
||||
"The `dict` method is deprecated; use `model_dump` instead",
|
||||
"Pydantic serializer warnings",
|
||||
"PydanticSerializationUnexpectedValue",
|
||||
"Expected 9 fields but got 5 for type `Message`",
|
||||
"Expected", # Catches all "Expected N fields but got M" patterns
|
||||
"Expected `StreamingChoices` but got `Choices`",
|
||||
"serialized value may not be as expected",
|
||||
"Mixing V1 models and V2 models",
|
||||
"Please upgrade `Settings` to V2"
|
||||
"Please upgrade `Settings` to V2",
|
||||
"There is no current event loop"
|
||||
]
|
||||
|
||||
@functools.wraps(_original_warn)
|
||||
@@ -43,7 +45,7 @@ def _patched_warn(message, category=None, stacklevel=1, source=None):
|
||||
if pattern in msg_str:
|
||||
return
|
||||
|
||||
if category == UserWarning and "pydantic" in msg_str.lower():
|
||||
if category is UserWarning and "pydantic" in msg_str.lower():
|
||||
return
|
||||
|
||||
_original_warn(message, category, stacklevel, source)
|
||||
@@ -57,7 +59,7 @@ def _patched_warn_explicit(message, category, filename, lineno, module=None, reg
|
||||
if pattern in msg_str:
|
||||
return
|
||||
|
||||
if category == UserWarning and "pydantic" in msg_str.lower():
|
||||
if category is UserWarning and "pydantic" in msg_str.lower():
|
||||
return
|
||||
|
||||
if module and "pydantic" in str(module):
|
||||
|
||||
@@ -250,7 +250,9 @@ class Agent:
|
||||
fast_context_parallelism: int = 8,
|
||||
fast_context_timeout: float = 30.0,
|
||||
history_in_context: Optional[int] = None,
|
||||
auto_save: Optional[str] = None
|
||||
auto_save: Optional[str] = None,
|
||||
skills: Optional[List[str]] = None,
|
||||
skills_dirs: Optional[List[str]] = None
|
||||
):
|
||||
"""Initialize an Agent instance.
|
||||
|
||||
@@ -607,6 +609,12 @@ Your Goal: {self.goal}
|
||||
self.fast_context_timeout = fast_context_timeout
|
||||
self._fast_context_instance = None # Lazy loaded
|
||||
|
||||
# Agent Skills configuration (lazy loaded for zero performance impact)
|
||||
self._skills = skills
|
||||
self._skills_dirs = skills_dirs
|
||||
self._skill_manager = None # Lazy loaded
|
||||
self._skills_initialized = False
|
||||
|
||||
@property
|
||||
def console(self):
|
||||
"""Lazily initialize Rich Console only when needed."""
|
||||
@@ -626,6 +634,72 @@ Your Goal: {self.goal}
|
||||
def fast_context_path(self, value):
|
||||
self._fast_context_path = value
|
||||
|
||||
@property
|
||||
def skill_manager(self):
|
||||
"""Lazily initialize SkillManager only when skills are accessed."""
|
||||
if self._skill_manager is None and (self._skills or self._skills_dirs):
|
||||
from ..skills import SkillManager
|
||||
self._skill_manager = SkillManager()
|
||||
|
||||
# Add explicit skill paths
|
||||
if self._skills:
|
||||
for skill_path in self._skills:
|
||||
self._skill_manager.add_skill(skill_path)
|
||||
|
||||
# Discover skills from directories
|
||||
if self._skills_dirs:
|
||||
self._skill_manager.discover(self._skills_dirs, include_defaults=False)
|
||||
|
||||
self._skills_initialized = True
|
||||
|
||||
# Auto-add skill execution tools if not already present
|
||||
self._add_skill_tools()
|
||||
return self._skill_manager
|
||||
|
||||
def _add_skill_tools(self):
|
||||
"""Add tools required for skill execution (read_file, run_skill_script).
|
||||
|
||||
Uses lazy imports from praisonaiagents.tools to avoid performance impact
|
||||
when skills are not used.
|
||||
"""
|
||||
# Check if tools already include required capabilities
|
||||
tool_names = set()
|
||||
for tool in self.tools:
|
||||
if callable(tool) and hasattr(tool, '__name__'):
|
||||
tool_names.add(tool.__name__)
|
||||
elif hasattr(tool, 'name'):
|
||||
tool_names.add(tool.name)
|
||||
|
||||
# Add read_file if not present
|
||||
if 'read_file' not in tool_names:
|
||||
try:
|
||||
from ..tools import read_file
|
||||
self.tools.append(read_file)
|
||||
logging.debug("Added read_file tool for skill support")
|
||||
except ImportError:
|
||||
logging.warning("Could not import read_file tool for skills")
|
||||
|
||||
# Add run_skill_script from skill_tools module
|
||||
if 'run_skill_script' not in tool_names:
|
||||
try:
|
||||
from ..tools.skill_tools import create_skill_tools
|
||||
# Create skill tools with current working directory
|
||||
skill_tools = create_skill_tools()
|
||||
self.tools.append(skill_tools.run_skill_script)
|
||||
logging.debug("Added run_skill_script tool for skill support")
|
||||
except ImportError as e:
|
||||
logging.warning(f"Could not import skill_tools: {e}")
|
||||
|
||||
def get_skills_prompt(self) -> str:
|
||||
"""Get the XML prompt for available skills.
|
||||
|
||||
Returns:
|
||||
XML string with <available_skills> block, or empty string if no skills
|
||||
"""
|
||||
if self.skill_manager is None:
|
||||
return ""
|
||||
return self.skill_manager.to_prompt()
|
||||
|
||||
@property
|
||||
def _openai_client(self):
|
||||
"""Lazily initialize OpenAI client only when needed."""
|
||||
@@ -1284,6 +1358,13 @@ Your Goal: {self.goal}"""
|
||||
if self.verbose:
|
||||
self._display_memory_info()
|
||||
|
||||
# Add skills prompt if skills are configured
|
||||
if self._skills or self._skills_dirs:
|
||||
skills_prompt = self.get_skills_prompt()
|
||||
if skills_prompt:
|
||||
system_prompt += f"\n\n## Available Skills\n{skills_prompt}"
|
||||
system_prompt += "\n\nWhen a skill is relevant to the task, read its SKILL.md file to get detailed instructions. If the skill has scripts in its scripts/ directory, you can execute them using the execute_code or run_script tool."
|
||||
|
||||
# Add tool usage instructions if tools are available
|
||||
# Use provided tools or fall back to self.tools
|
||||
tools_to_use = tools if tools is not None else self.tools
|
||||
@@ -1538,13 +1619,13 @@ Your Goal: {self.goal}"""
|
||||
logging.debug(f"{self.name} executing tool {function_name} with arguments: {arguments}")
|
||||
|
||||
# Check if approval is required for this tool
|
||||
from ..approval import is_approval_required, console_approval_callback, get_risk_level, mark_approved, ApprovalDecision
|
||||
from ..approval import is_approval_required, console_approval_callback, get_risk_level, mark_approved, ApprovalDecision, get_approval_callback
|
||||
if is_approval_required(function_name):
|
||||
risk_level = get_risk_level(function_name)
|
||||
logging.info(f"Tool {function_name} requires approval (risk level: {risk_level})")
|
||||
logging.debug(f"Tool {function_name} requires approval (risk level: {risk_level})")
|
||||
|
||||
# Use global approval callback or default console callback
|
||||
callback = approval_callback or console_approval_callback
|
||||
callback = get_approval_callback() or console_approval_callback
|
||||
|
||||
try:
|
||||
decision = callback(function_name, arguments, risk_level)
|
||||
|
||||
@@ -43,6 +43,15 @@ def set_approval_callback(callback_fn: Callable):
|
||||
global approval_callback
|
||||
approval_callback = callback_fn
|
||||
|
||||
def get_approval_callback() -> Optional[Callable]:
|
||||
"""Get the current approval callback function.
|
||||
|
||||
Returns the custom callback if set, otherwise None.
|
||||
This should be used instead of directly accessing approval_callback
|
||||
to ensure the latest callback is always used.
|
||||
"""
|
||||
return approval_callback
|
||||
|
||||
def mark_approved(tool_name: str):
|
||||
"""Mark a tool as approved in the current context."""
|
||||
approved = _approved_context.get(set())
|
||||
@@ -90,7 +99,7 @@ def require_approval(risk_level: RiskLevel = "high"):
|
||||
except Exception as e:
|
||||
# Fallback to sync approval if async fails
|
||||
logging.warning(f"Async approval failed, using sync fallback: {e}")
|
||||
callback = approval_callback or console_approval_callback
|
||||
callback = get_approval_callback() or console_approval_callback
|
||||
decision = callback(tool_name, kwargs, risk_level)
|
||||
|
||||
if not decision.approved:
|
||||
@@ -198,7 +207,7 @@ async def request_approval(function_name: str, arguments: Dict[str, Any]) -> App
|
||||
risk_level = TOOL_RISK_LEVELS.get(function_name, "medium")
|
||||
|
||||
# Use custom callback if set, otherwise use console callback
|
||||
callback = approval_callback or console_approval_callback
|
||||
callback = get_approval_callback() or console_approval_callback
|
||||
|
||||
try:
|
||||
# Handle async callbacks
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
"""
|
||||
PraisonAI Agents Evaluation Framework.
|
||||
|
||||
Provides comprehensive evaluation capabilities for AI agents with zero performance
|
||||
impact when not in use through lazy loading.
|
||||
|
||||
Evaluator Types:
|
||||
- AccuracyEvaluator: Compare output against expected output using LLM-as-judge
|
||||
- PerformanceEvaluator: Measure runtime and memory usage
|
||||
- ReliabilityEvaluator: Verify expected tool calls are made
|
||||
- CriteriaEvaluator: Evaluate against custom criteria
|
||||
|
||||
Example:
|
||||
>>> from praisonaiagents.eval import AccuracyEvaluator
|
||||
>>> evaluator = AccuracyEvaluator(
|
||||
... agent=my_agent,
|
||||
... input_text="What is 2+2?",
|
||||
... expected_output="4"
|
||||
... )
|
||||
>>> result = evaluator.run(print_summary=True)
|
||||
"""
|
||||
|
||||
__all__ = [
|
||||
"BaseEvaluator",
|
||||
"AccuracyEvaluator",
|
||||
"PerformanceEvaluator",
|
||||
"ReliabilityEvaluator",
|
||||
"CriteriaEvaluator",
|
||||
"AccuracyResult",
|
||||
"PerformanceResult",
|
||||
"ReliabilityResult",
|
||||
"CriteriaResult",
|
||||
"EvaluationScore",
|
||||
"PerformanceMetrics",
|
||||
"ToolCallResult",
|
||||
"CriteriaScore",
|
||||
]
|
||||
|
||||
_LAZY_IMPORTS = {
|
||||
"BaseEvaluator": ("base", "BaseEvaluator"),
|
||||
"AccuracyEvaluator": ("accuracy", "AccuracyEvaluator"),
|
||||
"PerformanceEvaluator": ("performance", "PerformanceEvaluator"),
|
||||
"ReliabilityEvaluator": ("reliability", "ReliabilityEvaluator"),
|
||||
"CriteriaEvaluator": ("criteria", "CriteriaEvaluator"),
|
||||
"AccuracyResult": ("results", "AccuracyResult"),
|
||||
"PerformanceResult": ("results", "PerformanceResult"),
|
||||
"ReliabilityResult": ("results", "ReliabilityResult"),
|
||||
"CriteriaResult": ("results", "CriteriaResult"),
|
||||
"EvaluationScore": ("results", "EvaluationScore"),
|
||||
"PerformanceMetrics": ("results", "PerformanceMetrics"),
|
||||
"ToolCallResult": ("results", "ToolCallResult"),
|
||||
"CriteriaScore": ("results", "CriteriaScore"),
|
||||
}
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
"""Lazy import mechanism for zero-cost imports when not used."""
|
||||
if name in _LAZY_IMPORTS:
|
||||
module_name, attr_name = _LAZY_IMPORTS[name]
|
||||
import importlib
|
||||
module = importlib.import_module(f".{module_name}", __name__)
|
||||
return getattr(module, attr_name)
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
def __dir__():
|
||||
"""Return list of available attributes for tab completion."""
|
||||
return __all__
|
||||
@@ -0,0 +1,239 @@
|
||||
"""
|
||||
Accuracy evaluator for PraisonAI Agents.
|
||||
|
||||
Evaluates agent output accuracy by comparing against expected output using LLM-as-judge.
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from typing import Any, Callable, List, Optional, Union, TYPE_CHECKING
|
||||
|
||||
from .base import BaseEvaluator
|
||||
from .results import AccuracyResult, EvaluationScore
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..agent.agent import Agent
|
||||
from ..agents.agents import PraisonAIAgents
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AccuracyEvaluator(BaseEvaluator):
|
||||
"""
|
||||
Evaluates the accuracy of agent outputs against expected outputs.
|
||||
|
||||
Uses an LLM as a judge to score outputs on a scale of 1-10.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
agent: Optional[Union["Agent", "PraisonAIAgents"]] = None,
|
||||
func: Optional[Callable[..., str]] = None,
|
||||
input_text: str = "",
|
||||
expected_output: str = "",
|
||||
num_iterations: int = 1,
|
||||
model: Optional[str] = None,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the accuracy evaluator.
|
||||
|
||||
Args:
|
||||
agent: Agent or PraisonAIAgents instance to evaluate
|
||||
func: Alternative callable that returns output string
|
||||
input_text: Input to provide to the agent/function
|
||||
expected_output: Expected output to compare against
|
||||
num_iterations: Number of evaluation iterations
|
||||
model: LLM model to use for judging (defaults to gpt-4o-mini)
|
||||
name: Name for this evaluation
|
||||
save_results_path: Path to save results
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
super().__init__(name=name, save_results_path=save_results_path, verbose=verbose)
|
||||
|
||||
self.agent = agent
|
||||
self.func = func
|
||||
self.input_text = input_text
|
||||
self.expected_output = expected_output
|
||||
self.num_iterations = num_iterations
|
||||
self.model = model or os.getenv("OPENAI_MODEL_NAME", "gpt-4o-mini")
|
||||
|
||||
if agent is None and func is None:
|
||||
raise ValueError("Either 'agent' or 'func' must be provided")
|
||||
|
||||
def _get_output(self) -> str:
|
||||
"""Get output from agent or function."""
|
||||
if self.func:
|
||||
return str(self.func(self.input_text))
|
||||
|
||||
if hasattr(self.agent, 'chat'):
|
||||
return str(self.agent.chat(self.input_text))
|
||||
elif hasattr(self.agent, 'start'):
|
||||
result = self.agent.start(self.input_text)
|
||||
if hasattr(result, 'raw'):
|
||||
return str(result.raw)
|
||||
return str(result)
|
||||
else:
|
||||
raise ValueError("Agent must have 'chat' or 'start' method")
|
||||
|
||||
def _judge_output(self, output: str) -> EvaluationScore:
|
||||
"""
|
||||
Use LLM to judge the output against expected output.
|
||||
|
||||
Args:
|
||||
output: The actual output to evaluate
|
||||
|
||||
Returns:
|
||||
EvaluationScore with score and reasoning
|
||||
"""
|
||||
try:
|
||||
from openai import OpenAI
|
||||
except ImportError:
|
||||
raise ImportError("OpenAI package required for accuracy evaluation. Install with: pip install openai")
|
||||
|
||||
client = OpenAI()
|
||||
|
||||
prompt = f"""You are an expert evaluator. Compare the actual output against the expected output and provide a score from 1-10.
|
||||
|
||||
Input: {self.input_text}
|
||||
|
||||
Expected Output: {self.expected_output}
|
||||
|
||||
Actual Output: {output}
|
||||
|
||||
Scoring Guidelines:
|
||||
- 10: Perfect match in meaning and completeness
|
||||
- 8-9: Very close, minor differences that don't affect correctness
|
||||
- 6-7: Mostly correct but missing some details or has minor errors
|
||||
- 4-5: Partially correct but significant issues
|
||||
- 2-3: Mostly incorrect but shows some understanding
|
||||
- 1: Completely wrong or irrelevant
|
||||
|
||||
Respond in this exact format:
|
||||
SCORE: [number 1-10]
|
||||
REASONING: [brief explanation]"""
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.1,
|
||||
max_tokens=200
|
||||
)
|
||||
|
||||
response_text = response.choices[0].message.content or ""
|
||||
|
||||
score = 5.0
|
||||
reasoning = "Unable to parse response"
|
||||
|
||||
lines = response_text.strip().split('\n')
|
||||
for line in lines:
|
||||
if line.startswith('SCORE:'):
|
||||
try:
|
||||
score = float(line.replace('SCORE:', '').strip())
|
||||
score = max(1.0, min(10.0, score))
|
||||
except ValueError:
|
||||
pass
|
||||
elif line.startswith('REASONING:'):
|
||||
reasoning = line.replace('REASONING:', '').strip()
|
||||
|
||||
return EvaluationScore(
|
||||
score=score,
|
||||
reasoning=reasoning,
|
||||
input_text=self.input_text,
|
||||
output_text=output,
|
||||
expected_output=self.expected_output
|
||||
)
|
||||
|
||||
def run(self, print_summary: bool = False) -> AccuracyResult:
|
||||
"""
|
||||
Execute the accuracy evaluation.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
AccuracyResult with all evaluation scores
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = AccuracyResult(
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
for i in range(self.num_iterations):
|
||||
if self.verbose:
|
||||
logger.info(f"Running iteration {i + 1}/{self.num_iterations}")
|
||||
|
||||
try:
|
||||
output = self._get_output()
|
||||
score = self._judge_output(output)
|
||||
result.evaluations.append(score)
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f" Score: {score.score}/10 - {score.reasoning}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error in iteration {i + 1}: {e}")
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=0.0,
|
||||
reasoning=f"Error: {str(e)}",
|
||||
input_text=self.input_text,
|
||||
output_text="",
|
||||
expected_output=self.expected_output
|
||||
))
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
async def run_async(self, print_summary: bool = False) -> AccuracyResult:
|
||||
"""
|
||||
Execute the accuracy evaluation asynchronously.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
AccuracyResult with all evaluation scores
|
||||
"""
|
||||
await self.async_before_run()
|
||||
result = self.run(print_summary=False)
|
||||
await self.async_after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
def evaluate_output(self, output: str, print_summary: bool = False) -> AccuracyResult:
|
||||
"""
|
||||
Evaluate a pre-generated output without running the agent.
|
||||
|
||||
Args:
|
||||
output: The output to evaluate
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
AccuracyResult with evaluation score
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = AccuracyResult(
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
score = self._judge_output(output)
|
||||
result.evaluations.append(score)
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,143 @@
|
||||
"""
|
||||
Base evaluator class for PraisonAI Agents evaluation framework.
|
||||
|
||||
This module provides the abstract base class that all evaluators inherit from.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Dict, Optional, Union, TYPE_CHECKING
|
||||
import uuid
|
||||
import logging
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .results import AccuracyResult, PerformanceResult, ReliabilityResult, CriteriaResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BaseEvaluator(ABC):
|
||||
"""
|
||||
Abstract base class for all evaluators in PraisonAI Agents.
|
||||
|
||||
Provides common functionality for evaluation lifecycle management,
|
||||
result storage, and telemetry integration.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the base evaluator.
|
||||
|
||||
Args:
|
||||
name: Optional name for this evaluation run
|
||||
save_results_path: Optional file path to save results (supports {name}, {eval_id} placeholders)
|
||||
verbose: Enable verbose output during evaluation
|
||||
"""
|
||||
self.eval_id = str(uuid.uuid4())[:8]
|
||||
self.name = name or f"eval_{self.eval_id}"
|
||||
self.save_results_path = save_results_path
|
||||
self.verbose = verbose
|
||||
self._result = None
|
||||
|
||||
def before_run(self) -> None:
|
||||
"""
|
||||
Hook called before evaluation runs.
|
||||
Override in subclasses for custom pre-evaluation logic.
|
||||
"""
|
||||
if self.verbose:
|
||||
logger.info(f"Starting evaluation: {self.name} (ID: {self.eval_id})")
|
||||
|
||||
def after_run(self, result: Any) -> None:
|
||||
"""
|
||||
Hook called after evaluation completes.
|
||||
Override in subclasses for custom post-evaluation logic.
|
||||
|
||||
Args:
|
||||
result: The evaluation result
|
||||
"""
|
||||
if self.verbose:
|
||||
logger.info(f"Completed evaluation: {self.name}")
|
||||
|
||||
if self.save_results_path:
|
||||
self._save_result(result)
|
||||
|
||||
async def async_before_run(self) -> None:
|
||||
"""
|
||||
Async hook called before evaluation runs.
|
||||
Override in subclasses for custom async pre-evaluation logic.
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
async def async_after_run(self, result: Any) -> None:
|
||||
"""
|
||||
Async hook called after evaluation completes.
|
||||
Override in subclasses for custom async post-evaluation logic.
|
||||
|
||||
Args:
|
||||
result: The evaluation result
|
||||
"""
|
||||
self.after_run(result)
|
||||
|
||||
@abstractmethod
|
||||
def run(self, **kwargs) -> Any:
|
||||
"""
|
||||
Execute the evaluation synchronously.
|
||||
|
||||
Returns:
|
||||
Evaluation result (type depends on evaluator)
|
||||
"""
|
||||
pass
|
||||
|
||||
async def run_async(self, **kwargs) -> Any:
|
||||
"""
|
||||
Execute the evaluation asynchronously.
|
||||
Default implementation calls sync run().
|
||||
Override for true async evaluation.
|
||||
|
||||
Returns:
|
||||
Evaluation result (type depends on evaluator)
|
||||
"""
|
||||
return self.run(**kwargs)
|
||||
|
||||
def _save_result(self, result: Any) -> None:
|
||||
"""
|
||||
Save evaluation result to file.
|
||||
|
||||
Args:
|
||||
result: The evaluation result to save
|
||||
"""
|
||||
if not self.save_results_path:
|
||||
return
|
||||
|
||||
try:
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
file_path = self.save_results_path.format(
|
||||
name=self.name,
|
||||
eval_id=self.eval_id
|
||||
)
|
||||
|
||||
path = Path(file_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if hasattr(result, 'to_dict'):
|
||||
data = result.to_dict()
|
||||
elif hasattr(result, '__dict__'):
|
||||
data = result.__dict__
|
||||
else:
|
||||
data = {"result": str(result)}
|
||||
|
||||
path.write_text(json.dumps(data, indent=2, default=str))
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f"Saved results to: {file_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save results: {e}")
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"{self.__class__.__name__}(name={self.name!r}, eval_id={self.eval_id!r})"
|
||||
@@ -0,0 +1,289 @@
|
||||
"""
|
||||
Criteria evaluator for PraisonAI Agents.
|
||||
|
||||
Evaluates agent outputs against custom criteria using LLM-as-judge.
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from typing import Callable, List, Literal, Optional, Union, TYPE_CHECKING
|
||||
|
||||
from .base import BaseEvaluator
|
||||
from .results import CriteriaResult, CriteriaScore
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..agent.agent import Agent
|
||||
from ..agents.agents import PraisonAIAgents
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CriteriaEvaluator(BaseEvaluator):
|
||||
"""
|
||||
Evaluates agent outputs against custom criteria.
|
||||
|
||||
Uses an LLM as a judge to score outputs based on user-defined criteria.
|
||||
Supports both numeric (1-10) and binary (pass/fail) scoring.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
criteria: str,
|
||||
agent: Optional[Union["Agent", "PraisonAIAgents"]] = None,
|
||||
func: Optional[Callable[..., str]] = None,
|
||||
input_text: str = "",
|
||||
scoring_type: Literal["numeric", "binary"] = "numeric",
|
||||
threshold: float = 7.0,
|
||||
num_iterations: int = 1,
|
||||
model: Optional[str] = None,
|
||||
on_fail: Optional[Callable[[CriteriaScore], None]] = None,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the criteria evaluator.
|
||||
|
||||
Args:
|
||||
criteria: The criteria to evaluate against (e.g., "Response is helpful and accurate")
|
||||
agent: Agent or PraisonAIAgents instance to evaluate
|
||||
func: Alternative callable that returns output string
|
||||
input_text: Input to provide to the agent/function
|
||||
scoring_type: "numeric" (1-10 scale) or "binary" (pass/fail)
|
||||
threshold: Score threshold for passing (numeric mode only)
|
||||
num_iterations: Number of evaluation iterations
|
||||
model: LLM model to use for judging (defaults to gpt-4o-mini)
|
||||
on_fail: Callback function called when evaluation fails
|
||||
name: Name for this evaluation
|
||||
save_results_path: Path to save results
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
super().__init__(name=name, save_results_path=save_results_path, verbose=verbose)
|
||||
|
||||
self.criteria = criteria
|
||||
self.agent = agent
|
||||
self.func = func
|
||||
self.input_text = input_text
|
||||
self.scoring_type = scoring_type
|
||||
self.threshold = threshold
|
||||
self.num_iterations = num_iterations
|
||||
self.model = model or os.getenv("OPENAI_MODEL_NAME", "gpt-4o-mini")
|
||||
self.on_fail = on_fail
|
||||
|
||||
def _get_output(self) -> str:
|
||||
"""Get output from agent or function."""
|
||||
if self.func:
|
||||
return str(self.func(self.input_text))
|
||||
|
||||
if self.agent is None:
|
||||
raise ValueError("Either 'agent' or 'func' must be provided")
|
||||
|
||||
if hasattr(self.agent, 'chat'):
|
||||
return str(self.agent.chat(self.input_text))
|
||||
elif hasattr(self.agent, 'start'):
|
||||
result = self.agent.start(self.input_text)
|
||||
if hasattr(result, 'raw'):
|
||||
return str(result.raw)
|
||||
return str(result)
|
||||
else:
|
||||
raise ValueError("Agent must have 'chat' or 'start' method")
|
||||
|
||||
def _judge_output(self, output: str) -> CriteriaScore:
|
||||
"""
|
||||
Use LLM to judge the output against criteria.
|
||||
|
||||
Args:
|
||||
output: The output to evaluate
|
||||
|
||||
Returns:
|
||||
CriteriaScore with score, passed status, and reasoning
|
||||
"""
|
||||
try:
|
||||
from openai import OpenAI
|
||||
except ImportError:
|
||||
raise ImportError("OpenAI package required for criteria evaluation. Install with: pip install openai")
|
||||
|
||||
client = OpenAI()
|
||||
|
||||
if self.scoring_type == "binary":
|
||||
prompt = f"""You are an expert evaluator. Evaluate the following output against the given criteria.
|
||||
|
||||
Criteria: {self.criteria}
|
||||
|
||||
Output to evaluate:
|
||||
{output}
|
||||
|
||||
Determine if the output PASSES or FAILS the criteria.
|
||||
|
||||
Respond in this exact format:
|
||||
RESULT: [PASS or FAIL]
|
||||
REASONING: [brief explanation]"""
|
||||
else:
|
||||
prompt = f"""You are an expert evaluator. Evaluate the following output against the given criteria.
|
||||
|
||||
Criteria: {self.criteria}
|
||||
|
||||
Output to evaluate:
|
||||
{output}
|
||||
|
||||
Score the output from 1-10 based on how well it meets the criteria.
|
||||
- 10: Perfectly meets all criteria
|
||||
- 8-9: Meets criteria very well with minor issues
|
||||
- 6-7: Meets most criteria but has some gaps
|
||||
- 4-5: Partially meets criteria
|
||||
- 2-3: Barely meets criteria
|
||||
- 1: Does not meet criteria at all
|
||||
|
||||
Respond in this exact format:
|
||||
SCORE: [number 1-10]
|
||||
REASONING: [brief explanation]"""
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.1,
|
||||
max_tokens=200
|
||||
)
|
||||
|
||||
response_text = response.choices[0].message.content or ""
|
||||
|
||||
score = 5.0
|
||||
passed = False
|
||||
reasoning = "Unable to parse response"
|
||||
|
||||
lines = response_text.strip().split('\n')
|
||||
for line in lines:
|
||||
if self.scoring_type == "binary":
|
||||
if line.startswith('RESULT:'):
|
||||
result_str = line.replace('RESULT:', '').strip().upper()
|
||||
passed = result_str == 'PASS'
|
||||
score = 10.0 if passed else 0.0
|
||||
else:
|
||||
if line.startswith('SCORE:'):
|
||||
try:
|
||||
score = float(line.replace('SCORE:', '').strip())
|
||||
score = max(1.0, min(10.0, score))
|
||||
passed = score >= self.threshold
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if line.startswith('REASONING:'):
|
||||
reasoning = line.replace('REASONING:', '').strip()
|
||||
|
||||
return CriteriaScore(
|
||||
score=score,
|
||||
passed=passed,
|
||||
reasoning=reasoning,
|
||||
output_text=output,
|
||||
criteria=self.criteria
|
||||
)
|
||||
|
||||
def run(self, print_summary: bool = False) -> CriteriaResult:
|
||||
"""
|
||||
Execute the criteria evaluation.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
CriteriaResult with all evaluation scores
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = CriteriaResult(
|
||||
criteria=self.criteria,
|
||||
scoring_type=self.scoring_type,
|
||||
threshold=self.threshold,
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
for i in range(self.num_iterations):
|
||||
if self.verbose:
|
||||
logger.info(f"Running iteration {i + 1}/{self.num_iterations}")
|
||||
|
||||
try:
|
||||
output = self._get_output()
|
||||
score = self._judge_output(output)
|
||||
result.evaluations.append(score)
|
||||
|
||||
if self.verbose:
|
||||
status = "PASS" if score.passed else "FAIL"
|
||||
logger.info(f" {status} (Score: {score.score}) - {score.reasoning}")
|
||||
|
||||
if not score.passed and self.on_fail:
|
||||
self.on_fail(score)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in iteration {i + 1}: {e}")
|
||||
error_score = CriteriaScore(
|
||||
score=0.0,
|
||||
passed=False,
|
||||
reasoning=f"Error: {str(e)}",
|
||||
output_text="",
|
||||
criteria=self.criteria
|
||||
)
|
||||
result.evaluations.append(error_score)
|
||||
|
||||
if self.on_fail:
|
||||
self.on_fail(error_score)
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
async def run_async(self, print_summary: bool = False) -> CriteriaResult:
|
||||
"""
|
||||
Execute the criteria evaluation asynchronously.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
CriteriaResult with all evaluation scores
|
||||
"""
|
||||
await self.async_before_run()
|
||||
result = self.run(print_summary=False)
|
||||
await self.async_after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
def evaluate_output(self, output: str, print_summary: bool = False) -> CriteriaResult:
|
||||
"""
|
||||
Evaluate a pre-generated output without running the agent.
|
||||
|
||||
Args:
|
||||
output: The output to evaluate
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
CriteriaResult with evaluation score
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = CriteriaResult(
|
||||
criteria=self.criteria,
|
||||
scoring_type=self.scoring_type,
|
||||
threshold=self.threshold,
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
score = self._judge_output(output)
|
||||
result.evaluations.append(score)
|
||||
|
||||
if not score.passed and self.on_fail:
|
||||
self.on_fail(score)
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,184 @@
|
||||
"""
|
||||
Performance evaluator for PraisonAI Agents.
|
||||
|
||||
Measures runtime and memory usage of agent executions or functions.
|
||||
"""
|
||||
|
||||
import time
|
||||
import logging
|
||||
from typing import Callable, Optional, TYPE_CHECKING
|
||||
|
||||
from .base import BaseEvaluator
|
||||
from .results import PerformanceResult, PerformanceMetrics
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..agent.agent import Agent
|
||||
from ..agents.agents import PraisonAIAgents
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PerformanceEvaluator(BaseEvaluator):
|
||||
"""
|
||||
Evaluates the performance of agent executions or functions.
|
||||
|
||||
Measures runtime and memory usage across multiple iterations.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
func: Optional[Callable[[], None]] = None,
|
||||
agent: Optional["Agent"] = None,
|
||||
input_text: str = "",
|
||||
num_iterations: int = 10,
|
||||
warmup_runs: int = 2,
|
||||
track_memory: bool = True,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the performance evaluator.
|
||||
|
||||
Args:
|
||||
func: Callable to benchmark (takes no arguments)
|
||||
agent: Agent to benchmark (will call chat with input_text)
|
||||
input_text: Input text for agent benchmarking
|
||||
num_iterations: Number of benchmark iterations
|
||||
warmup_runs: Number of warmup runs before measurement
|
||||
track_memory: Whether to track memory usage
|
||||
name: Name for this evaluation
|
||||
save_results_path: Path to save results
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
super().__init__(name=name, save_results_path=save_results_path, verbose=verbose)
|
||||
|
||||
self.func = func
|
||||
self.agent = agent
|
||||
self.input_text = input_text
|
||||
self.num_iterations = num_iterations
|
||||
self.warmup_runs = warmup_runs
|
||||
self.track_memory = track_memory
|
||||
|
||||
if func is None and agent is None:
|
||||
raise ValueError("Either 'func' or 'agent' must be provided")
|
||||
|
||||
def _get_memory_usage(self) -> float:
|
||||
"""Get current memory usage in MB."""
|
||||
if not self.track_memory:
|
||||
return 0.0
|
||||
|
||||
try:
|
||||
import tracemalloc
|
||||
current, _ = tracemalloc.get_traced_memory()
|
||||
return current / (1024 * 1024)
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
def _execute(self) -> None:
|
||||
"""Execute the function or agent."""
|
||||
if self.func:
|
||||
self.func()
|
||||
elif self.agent:
|
||||
if hasattr(self.agent, 'chat'):
|
||||
self.agent.chat(self.input_text)
|
||||
elif hasattr(self.agent, 'start'):
|
||||
self.agent.start(self.input_text)
|
||||
else:
|
||||
raise ValueError("Agent must have 'chat' or 'start' method")
|
||||
|
||||
def run(self, print_summary: bool = False) -> PerformanceResult:
|
||||
"""
|
||||
Execute the performance evaluation.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
PerformanceResult with all performance metrics
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = PerformanceResult(
|
||||
warmup_runs=self.warmup_runs,
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
if self.track_memory:
|
||||
try:
|
||||
import tracemalloc
|
||||
tracemalloc.start()
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not start memory tracking: {e}")
|
||||
self.track_memory = False
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f"Running {self.warmup_runs} warmup iterations...")
|
||||
|
||||
for i in range(self.warmup_runs):
|
||||
try:
|
||||
self._execute()
|
||||
except Exception as e:
|
||||
logger.warning(f"Warmup iteration {i + 1} failed: {e}")
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f"Running {self.num_iterations} benchmark iterations...")
|
||||
|
||||
for i in range(self.num_iterations):
|
||||
memory_before = self._get_memory_usage()
|
||||
|
||||
start_time = time.perf_counter()
|
||||
try:
|
||||
self._execute()
|
||||
success = True
|
||||
except Exception as e:
|
||||
logger.error(f"Iteration {i + 1} failed: {e}")
|
||||
success = False
|
||||
end_time = time.perf_counter()
|
||||
|
||||
memory_after = self._get_memory_usage()
|
||||
|
||||
if success:
|
||||
metrics = PerformanceMetrics(
|
||||
run_time=end_time - start_time,
|
||||
memory_usage=max(0, memory_after - memory_before),
|
||||
iteration=i + 1
|
||||
)
|
||||
result.metrics.append(metrics)
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f" Iteration {i + 1}: {metrics.run_time:.4f}s, {metrics.memory_usage:.2f}MB")
|
||||
|
||||
if self.track_memory:
|
||||
try:
|
||||
import tracemalloc
|
||||
tracemalloc.stop()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
async def run_async(self, print_summary: bool = False) -> PerformanceResult:
|
||||
"""
|
||||
Execute the performance evaluation asynchronously.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
PerformanceResult with all performance metrics
|
||||
"""
|
||||
await self.async_before_run()
|
||||
result = self.run(print_summary=False)
|
||||
await self.async_after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,225 @@
|
||||
"""
|
||||
Reliability evaluator for PraisonAI Agents.
|
||||
|
||||
Evaluates agent reliability by verifying expected tool calls are made.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Dict, List, Optional, Set, Any, TYPE_CHECKING
|
||||
|
||||
from .base import BaseEvaluator
|
||||
from .results import ReliabilityResult, ToolCallResult
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..agent.agent import Agent
|
||||
from ..agents.agents import PraisonAIAgents
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ReliabilityEvaluator(BaseEvaluator):
|
||||
"""
|
||||
Evaluates the reliability of agent tool usage.
|
||||
|
||||
Verifies that expected tools are called during agent execution.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
agent: Optional["Agent"] = None,
|
||||
input_text: str = "",
|
||||
expected_tools: Optional[List[str]] = None,
|
||||
forbidden_tools: Optional[List[str]] = None,
|
||||
name: Optional[str] = None,
|
||||
save_results_path: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the reliability evaluator.
|
||||
|
||||
Args:
|
||||
agent: Agent to evaluate
|
||||
input_text: Input to provide to the agent
|
||||
expected_tools: List of tool names that should be called
|
||||
forbidden_tools: List of tool names that should NOT be called
|
||||
name: Name for this evaluation
|
||||
save_results_path: Path to save results
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
super().__init__(name=name, save_results_path=save_results_path, verbose=verbose)
|
||||
|
||||
self.agent = agent
|
||||
self.input_text = input_text
|
||||
self.expected_tools = set(expected_tools or [])
|
||||
self.forbidden_tools = set(forbidden_tools or [])
|
||||
|
||||
if agent is None:
|
||||
raise ValueError("'agent' must be provided")
|
||||
|
||||
def _extract_tool_calls(self, response: Any) -> Set[str]:
|
||||
"""
|
||||
Extract tool names from agent response.
|
||||
|
||||
Args:
|
||||
response: Agent response object
|
||||
|
||||
Returns:
|
||||
Set of tool names that were called
|
||||
"""
|
||||
tool_calls = set()
|
||||
|
||||
if hasattr(response, 'tool_calls') and response.tool_calls:
|
||||
for tc in response.tool_calls:
|
||||
if hasattr(tc, 'name'):
|
||||
tool_calls.add(tc.name)
|
||||
elif hasattr(tc, 'function') and hasattr(tc.function, 'name'):
|
||||
tool_calls.add(tc.function.name)
|
||||
|
||||
if hasattr(self.agent, 'chat_history'):
|
||||
for msg in self.agent.chat_history:
|
||||
if isinstance(msg, dict):
|
||||
if msg.get('role') == 'assistant' and 'tool_calls' in msg:
|
||||
for tc in msg.get('tool_calls', []):
|
||||
if isinstance(tc, dict) and 'function' in tc:
|
||||
tool_calls.add(tc['function'].get('name', ''))
|
||||
elif msg.get('role') == 'tool':
|
||||
tool_name = msg.get('name', '')
|
||||
if tool_name:
|
||||
tool_calls.add(tool_name)
|
||||
|
||||
return tool_calls
|
||||
|
||||
def run(self, print_summary: bool = False) -> ReliabilityResult:
|
||||
"""
|
||||
Execute the reliability evaluation.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
ReliabilityResult with tool call verification results
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = ReliabilityResult(
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
try:
|
||||
if hasattr(self.agent, 'chat'):
|
||||
response = self.agent.chat(self.input_text)
|
||||
elif hasattr(self.agent, 'start'):
|
||||
response = self.agent.start(self.input_text)
|
||||
else:
|
||||
raise ValueError("Agent must have 'chat' or 'start' method")
|
||||
|
||||
actual_tools = self._extract_tool_calls(response)
|
||||
|
||||
if self.verbose:
|
||||
logger.info(f"Expected tools: {self.expected_tools}")
|
||||
logger.info(f"Actual tools called: {actual_tools}")
|
||||
|
||||
for tool in self.expected_tools:
|
||||
was_called = tool in actual_tools
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=True,
|
||||
actual=was_called,
|
||||
passed=was_called
|
||||
))
|
||||
|
||||
for tool in self.forbidden_tools:
|
||||
was_called = tool in actual_tools
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=False,
|
||||
actual=was_called,
|
||||
passed=not was_called
|
||||
))
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during reliability evaluation: {e}")
|
||||
for tool in self.expected_tools:
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=True,
|
||||
actual=False,
|
||||
passed=False,
|
||||
metadata={"error": str(e)}
|
||||
))
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
async def run_async(self, print_summary: bool = False) -> ReliabilityResult:
|
||||
"""
|
||||
Execute the reliability evaluation asynchronously.
|
||||
|
||||
Args:
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
ReliabilityResult with tool call verification results
|
||||
"""
|
||||
await self.async_before_run()
|
||||
result = self.run(print_summary=False)
|
||||
await self.async_after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
|
||||
def evaluate_tool_calls(
|
||||
self,
|
||||
actual_tools: List[str],
|
||||
print_summary: bool = False
|
||||
) -> ReliabilityResult:
|
||||
"""
|
||||
Evaluate a pre-recorded list of tool calls without running the agent.
|
||||
|
||||
Args:
|
||||
actual_tools: List of tool names that were called
|
||||
print_summary: Whether to print summary after evaluation
|
||||
|
||||
Returns:
|
||||
ReliabilityResult with tool call verification results
|
||||
"""
|
||||
self.before_run()
|
||||
|
||||
result = ReliabilityResult(
|
||||
eval_id=self.eval_id,
|
||||
name=self.name
|
||||
)
|
||||
|
||||
actual_set = set(actual_tools)
|
||||
|
||||
for tool in self.expected_tools:
|
||||
was_called = tool in actual_set
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=True,
|
||||
actual=was_called,
|
||||
passed=was_called
|
||||
))
|
||||
|
||||
for tool in self.forbidden_tools:
|
||||
was_called = tool in actual_set
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=tool,
|
||||
expected=False,
|
||||
actual=was_called,
|
||||
passed=not was_called
|
||||
))
|
||||
|
||||
self.after_run(result)
|
||||
|
||||
if print_summary:
|
||||
result.print_summary()
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,463 @@
|
||||
"""
|
||||
Result dataclasses for PraisonAI Agents evaluation framework.
|
||||
|
||||
This module provides structured result types for different evaluation methods.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from typing import List, Optional, Dict, Any
|
||||
from datetime import datetime
|
||||
import statistics
|
||||
import json
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvaluationScore:
|
||||
"""Individual evaluation score with reasoning."""
|
||||
score: float
|
||||
reasoning: str
|
||||
input_text: str
|
||||
output_text: str
|
||||
expected_output: Optional[str] = None
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AccuracyResult:
|
||||
"""Result from accuracy evaluation comparing output to expected output."""
|
||||
evaluations: List[EvaluationScore] = field(default_factory=list)
|
||||
eval_id: str = ""
|
||||
name: str = ""
|
||||
|
||||
@property
|
||||
def scores(self) -> List[float]:
|
||||
"""Get list of all scores."""
|
||||
return [e.score for e in self.evaluations]
|
||||
|
||||
@property
|
||||
def avg_score(self) -> float:
|
||||
"""Calculate average score."""
|
||||
if not self.scores:
|
||||
return 0.0
|
||||
return statistics.mean(self.scores)
|
||||
|
||||
@property
|
||||
def min_score(self) -> float:
|
||||
"""Get minimum score."""
|
||||
if not self.scores:
|
||||
return 0.0
|
||||
return min(self.scores)
|
||||
|
||||
@property
|
||||
def max_score(self) -> float:
|
||||
"""Get maximum score."""
|
||||
if not self.scores:
|
||||
return 0.0
|
||||
return max(self.scores)
|
||||
|
||||
@property
|
||||
def std_dev(self) -> float:
|
||||
"""Calculate standard deviation of scores."""
|
||||
if len(self.scores) < 2:
|
||||
return 0.0
|
||||
return statistics.stdev(self.scores)
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
"""Check if evaluation passed (avg score >= 7)."""
|
||||
return self.avg_score >= 7.0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary."""
|
||||
return {
|
||||
"eval_id": self.eval_id,
|
||||
"name": self.name,
|
||||
"evaluations": [asdict(e) for e in self.evaluations],
|
||||
"avg_score": self.avg_score,
|
||||
"min_score": self.min_score,
|
||||
"max_score": self.max_score,
|
||||
"std_dev": self.std_dev,
|
||||
"passed": self.passed,
|
||||
"num_evaluations": len(self.evaluations)
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Convert result to JSON string."""
|
||||
return json.dumps(self.to_dict(), indent=2)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print a summary of the evaluation results."""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
from rich.panel import Panel # noqa: F401
|
||||
|
||||
console = Console()
|
||||
|
||||
table = Table(title="Accuracy Evaluation Summary")
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Evaluations", str(len(self.evaluations)))
|
||||
table.add_row("Average Score", f"{self.avg_score:.2f}")
|
||||
table.add_row("Min Score", f"{self.min_score:.2f}")
|
||||
table.add_row("Max Score", f"{self.max_score:.2f}")
|
||||
table.add_row("Std Dev", f"{self.std_dev:.2f}")
|
||||
table.add_row("Status", "✅ PASSED" if self.passed else "❌ FAILED")
|
||||
|
||||
console.print(table)
|
||||
except ImportError:
|
||||
print("Accuracy Evaluation Summary")
|
||||
print(f" Evaluations: {len(self.evaluations)}")
|
||||
print(f" Average Score: {self.avg_score:.2f}")
|
||||
print(f" Min Score: {self.min_score:.2f}")
|
||||
print(f" Max Score: {self.max_score:.2f}")
|
||||
print(f" Std Dev: {self.std_dev:.2f}")
|
||||
print(f" Status: {'PASSED' if self.passed else 'FAILED'}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class PerformanceMetrics:
|
||||
"""Individual performance measurement."""
|
||||
run_time: float
|
||||
memory_usage: float
|
||||
iteration: int
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PerformanceResult:
|
||||
"""Result from performance evaluation measuring runtime and memory."""
|
||||
metrics: List[PerformanceMetrics] = field(default_factory=list)
|
||||
warmup_runs: int = 0
|
||||
eval_id: str = ""
|
||||
name: str = ""
|
||||
|
||||
@property
|
||||
def run_times(self) -> List[float]:
|
||||
"""Get list of all run times."""
|
||||
return [m.run_time for m in self.metrics]
|
||||
|
||||
@property
|
||||
def memory_usages(self) -> List[float]:
|
||||
"""Get list of all memory usages."""
|
||||
return [m.memory_usage for m in self.metrics]
|
||||
|
||||
@property
|
||||
def avg_run_time(self) -> float:
|
||||
"""Calculate average run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
return statistics.mean(self.run_times)
|
||||
|
||||
@property
|
||||
def min_run_time(self) -> float:
|
||||
"""Get minimum run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
return min(self.run_times)
|
||||
|
||||
@property
|
||||
def max_run_time(self) -> float:
|
||||
"""Get maximum run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
return max(self.run_times)
|
||||
|
||||
@property
|
||||
def std_dev_run_time(self) -> float:
|
||||
"""Calculate standard deviation of run times."""
|
||||
if len(self.run_times) < 2:
|
||||
return 0.0
|
||||
return statistics.stdev(self.run_times)
|
||||
|
||||
@property
|
||||
def median_run_time(self) -> float:
|
||||
"""Calculate median run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
return statistics.median(self.run_times)
|
||||
|
||||
@property
|
||||
def p95_run_time(self) -> float:
|
||||
"""Calculate 95th percentile run time."""
|
||||
if not self.run_times:
|
||||
return 0.0
|
||||
sorted_times = sorted(self.run_times)
|
||||
idx = int(len(sorted_times) * 0.95)
|
||||
return sorted_times[min(idx, len(sorted_times) - 1)]
|
||||
|
||||
@property
|
||||
def avg_memory(self) -> float:
|
||||
"""Calculate average memory usage."""
|
||||
if not self.memory_usages:
|
||||
return 0.0
|
||||
return statistics.mean(self.memory_usages)
|
||||
|
||||
@property
|
||||
def max_memory(self) -> float:
|
||||
"""Get maximum memory usage."""
|
||||
if not self.memory_usages:
|
||||
return 0.0
|
||||
return max(self.memory_usages)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary."""
|
||||
return {
|
||||
"eval_id": self.eval_id,
|
||||
"name": self.name,
|
||||
"metrics": [asdict(m) for m in self.metrics],
|
||||
"warmup_runs": self.warmup_runs,
|
||||
"avg_run_time": self.avg_run_time,
|
||||
"min_run_time": self.min_run_time,
|
||||
"max_run_time": self.max_run_time,
|
||||
"std_dev_run_time": self.std_dev_run_time,
|
||||
"median_run_time": self.median_run_time,
|
||||
"p95_run_time": self.p95_run_time,
|
||||
"avg_memory": self.avg_memory,
|
||||
"max_memory": self.max_memory,
|
||||
"num_iterations": len(self.metrics)
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Convert result to JSON string."""
|
||||
return json.dumps(self.to_dict(), indent=2)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print a summary of the performance results."""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
|
||||
table = Table(title="Performance Evaluation Summary")
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Iterations", str(len(self.metrics)))
|
||||
table.add_row("Warmup Runs", str(self.warmup_runs))
|
||||
table.add_row("Avg Run Time", f"{self.avg_run_time:.4f}s")
|
||||
table.add_row("Min Run Time", f"{self.min_run_time:.4f}s")
|
||||
table.add_row("Max Run Time", f"{self.max_run_time:.4f}s")
|
||||
table.add_row("Std Dev", f"{self.std_dev_run_time:.4f}s")
|
||||
table.add_row("Median", f"{self.median_run_time:.4f}s")
|
||||
table.add_row("P95", f"{self.p95_run_time:.4f}s")
|
||||
table.add_row("Avg Memory", f"{self.avg_memory:.2f} MB")
|
||||
table.add_row("Max Memory", f"{self.max_memory:.2f} MB")
|
||||
|
||||
console.print(table)
|
||||
except ImportError:
|
||||
print("Performance Evaluation Summary")
|
||||
print(f" Iterations: {len(self.metrics)}")
|
||||
print(f" Warmup Runs: {self.warmup_runs}")
|
||||
print(f" Avg Run Time: {self.avg_run_time:.4f}s")
|
||||
print(f" Min Run Time: {self.min_run_time:.4f}s")
|
||||
print(f" Max Run Time: {self.max_run_time:.4f}s")
|
||||
print(f" Std Dev: {self.std_dev_run_time:.4f}s")
|
||||
print(f" Median: {self.median_run_time:.4f}s")
|
||||
print(f" P95: {self.p95_run_time:.4f}s")
|
||||
print(f" Avg Memory: {self.avg_memory:.2f} MB")
|
||||
print(f" Max Memory: {self.max_memory:.2f} MB")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCallResult:
|
||||
"""Result of a single tool call verification."""
|
||||
tool_name: str
|
||||
expected: bool
|
||||
actual: bool
|
||||
passed: bool
|
||||
arguments: Optional[Dict[str, Any]] = None
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReliabilityResult:
|
||||
"""Result from reliability evaluation checking tool calls."""
|
||||
tool_results: List[ToolCallResult] = field(default_factory=list)
|
||||
eval_id: str = ""
|
||||
name: str = ""
|
||||
|
||||
@property
|
||||
def passed_calls(self) -> List[ToolCallResult]:
|
||||
"""Get list of passed tool calls."""
|
||||
return [t for t in self.tool_results if t.passed]
|
||||
|
||||
@property
|
||||
def failed_calls(self) -> List[ToolCallResult]:
|
||||
"""Get list of failed tool calls."""
|
||||
return [t for t in self.tool_results if not t.passed]
|
||||
|
||||
@property
|
||||
def pass_rate(self) -> float:
|
||||
"""Calculate pass rate."""
|
||||
if not self.tool_results:
|
||||
return 0.0
|
||||
return len(self.passed_calls) / len(self.tool_results)
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
"""Get overall status."""
|
||||
return "PASSED" if len(self.failed_calls) == 0 else "FAILED"
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
"""Check if all tool calls passed."""
|
||||
return len(self.failed_calls) == 0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary."""
|
||||
return {
|
||||
"eval_id": self.eval_id,
|
||||
"name": self.name,
|
||||
"tool_results": [asdict(t) for t in self.tool_results],
|
||||
"passed_count": len(self.passed_calls),
|
||||
"failed_count": len(self.failed_calls),
|
||||
"pass_rate": self.pass_rate,
|
||||
"status": self.status
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Convert result to JSON string."""
|
||||
return json.dumps(self.to_dict(), indent=2)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print a summary of the reliability results."""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
|
||||
table = Table(title="Reliability Evaluation Summary")
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Total Tool Calls", str(len(self.tool_results)))
|
||||
table.add_row("Passed", str(len(self.passed_calls)))
|
||||
table.add_row("Failed", str(len(self.failed_calls)))
|
||||
table.add_row("Pass Rate", f"{self.pass_rate:.1%}")
|
||||
table.add_row("Status", "✅ PASSED" if self.passed else "❌ FAILED")
|
||||
|
||||
console.print(table)
|
||||
|
||||
if self.failed_calls:
|
||||
fail_table = Table(title="Failed Tool Calls")
|
||||
fail_table.add_column("Tool", style="red")
|
||||
fail_table.add_column("Expected", style="yellow")
|
||||
fail_table.add_column("Actual", style="yellow")
|
||||
|
||||
for tc in self.failed_calls:
|
||||
fail_table.add_row(tc.tool_name, str(tc.expected), str(tc.actual))
|
||||
|
||||
console.print(fail_table)
|
||||
except ImportError:
|
||||
print("Reliability Evaluation Summary")
|
||||
print(f" Total Tool Calls: {len(self.tool_results)}")
|
||||
print(f" Passed: {len(self.passed_calls)}")
|
||||
print(f" Failed: {len(self.failed_calls)}")
|
||||
print(f" Pass Rate: {self.pass_rate:.1%}")
|
||||
print(f" Status: {self.status}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class CriteriaScore:
|
||||
"""Individual criteria evaluation score."""
|
||||
score: float
|
||||
passed: bool
|
||||
reasoning: str
|
||||
output_text: str
|
||||
criteria: str
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CriteriaResult:
|
||||
"""Result from criteria-based evaluation (agent-as-judge)."""
|
||||
evaluations: List[CriteriaScore] = field(default_factory=list)
|
||||
criteria: str = ""
|
||||
scoring_type: str = "numeric" # "numeric" or "binary"
|
||||
threshold: float = 7.0
|
||||
eval_id: str = ""
|
||||
name: str = ""
|
||||
|
||||
@property
|
||||
def scores(self) -> List[float]:
|
||||
"""Get list of all scores."""
|
||||
return [e.score for e in self.evaluations]
|
||||
|
||||
@property
|
||||
def avg_score(self) -> float:
|
||||
"""Calculate average score."""
|
||||
if not self.scores:
|
||||
return 0.0
|
||||
return statistics.mean(self.scores)
|
||||
|
||||
@property
|
||||
def pass_rate(self) -> float:
|
||||
"""Calculate pass rate."""
|
||||
if not self.evaluations:
|
||||
return 0.0
|
||||
passed = sum(1 for e in self.evaluations if e.passed)
|
||||
return passed / len(self.evaluations)
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
"""Check if evaluation passed overall."""
|
||||
if self.scoring_type == "binary":
|
||||
return self.pass_rate >= 0.5
|
||||
return self.avg_score >= self.threshold
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary."""
|
||||
return {
|
||||
"eval_id": self.eval_id,
|
||||
"name": self.name,
|
||||
"criteria": self.criteria,
|
||||
"scoring_type": self.scoring_type,
|
||||
"threshold": self.threshold,
|
||||
"evaluations": [asdict(e) for e in self.evaluations],
|
||||
"avg_score": self.avg_score,
|
||||
"pass_rate": self.pass_rate,
|
||||
"passed": self.passed,
|
||||
"num_evaluations": len(self.evaluations)
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Convert result to JSON string."""
|
||||
return json.dumps(self.to_dict(), indent=2)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print a summary of the criteria evaluation results."""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
|
||||
table = Table(title="Criteria Evaluation Summary")
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Criteria", self.criteria[:50] + "..." if len(self.criteria) > 50 else self.criteria)
|
||||
table.add_row("Scoring Type", self.scoring_type)
|
||||
table.add_row("Threshold", str(self.threshold))
|
||||
table.add_row("Evaluations", str(len(self.evaluations)))
|
||||
table.add_row("Average Score", f"{self.avg_score:.2f}")
|
||||
table.add_row("Pass Rate", f"{self.pass_rate:.1%}")
|
||||
table.add_row("Status", "✅ PASSED" if self.passed else "❌ FAILED")
|
||||
|
||||
console.print(table)
|
||||
except ImportError:
|
||||
print("Criteria Evaluation Summary")
|
||||
print(f" Criteria: {self.criteria[:50]}...")
|
||||
print(f" Scoring Type: {self.scoring_type}")
|
||||
print(f" Threshold: {self.threshold}")
|
||||
print(f" Evaluations: {len(self.evaluations)}")
|
||||
print(f" Average Score: {self.avg_score:.2f}")
|
||||
print(f" Pass Rate: {self.pass_rate:.1%}")
|
||||
print(f" Status: {'PASSED' if self.passed else 'FAILED'}")
|
||||
@@ -0,0 +1,154 @@
|
||||
"""
|
||||
Utility functions for PraisonAI Agents evaluation framework.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Union
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def save_result_to_file(
|
||||
result: Any,
|
||||
file_path: str,
|
||||
eval_id: Optional[str] = None,
|
||||
name: Optional[str] = None
|
||||
) -> bool:
|
||||
"""
|
||||
Save evaluation result to a file.
|
||||
|
||||
Args:
|
||||
result: The evaluation result to save
|
||||
file_path: Path to save the result (supports {name}, {eval_id} placeholders)
|
||||
eval_id: Optional eval ID for placeholder substitution
|
||||
name: Optional name for placeholder substitution
|
||||
|
||||
Returns:
|
||||
True if save was successful, False otherwise
|
||||
"""
|
||||
try:
|
||||
formatted_path = file_path.format(name=name or "", eval_id=eval_id or "")
|
||||
path = Path(formatted_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if hasattr(result, 'to_dict'):
|
||||
data = result.to_dict()
|
||||
elif hasattr(result, '__dict__'):
|
||||
data = result.__dict__
|
||||
else:
|
||||
data = {"result": str(result)}
|
||||
|
||||
path.write_text(json.dumps(data, indent=2, default=str))
|
||||
logger.debug(f"Saved result to: {formatted_path}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save result: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def load_result_from_file(file_path: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Load evaluation result from a file.
|
||||
|
||||
Args:
|
||||
file_path: Path to the result file
|
||||
|
||||
Returns:
|
||||
Dictionary containing the result data, or None if load failed
|
||||
"""
|
||||
try:
|
||||
path = Path(file_path)
|
||||
if not path.exists():
|
||||
logger.warning(f"Result file not found: {file_path}")
|
||||
return None
|
||||
|
||||
data = json.loads(path.read_text())
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load result: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def format_score(score: float, max_score: float = 10.0) -> str:
|
||||
"""
|
||||
Format a score for display.
|
||||
|
||||
Args:
|
||||
score: The score value
|
||||
max_score: Maximum possible score
|
||||
|
||||
Returns:
|
||||
Formatted score string
|
||||
"""
|
||||
return f"{score:.1f}/{max_score:.0f}"
|
||||
|
||||
|
||||
def format_percentage(value: float) -> str:
|
||||
"""
|
||||
Format a value as a percentage.
|
||||
|
||||
Args:
|
||||
value: Value between 0 and 1
|
||||
|
||||
Returns:
|
||||
Formatted percentage string
|
||||
"""
|
||||
return f"{value * 100:.1f}%"
|
||||
|
||||
|
||||
def format_duration(seconds: float) -> str:
|
||||
"""
|
||||
Format a duration in seconds for display.
|
||||
|
||||
Args:
|
||||
seconds: Duration in seconds
|
||||
|
||||
Returns:
|
||||
Formatted duration string
|
||||
"""
|
||||
if seconds < 0.001:
|
||||
return f"{seconds * 1000000:.0f}μs"
|
||||
elif seconds < 1:
|
||||
return f"{seconds * 1000:.2f}ms"
|
||||
elif seconds < 60:
|
||||
return f"{seconds:.2f}s"
|
||||
else:
|
||||
minutes = int(seconds // 60)
|
||||
secs = seconds % 60
|
||||
return f"{minutes}m {secs:.1f}s"
|
||||
|
||||
|
||||
def format_memory(mb: float) -> str:
|
||||
"""
|
||||
Format memory usage for display.
|
||||
|
||||
Args:
|
||||
mb: Memory in megabytes
|
||||
|
||||
Returns:
|
||||
Formatted memory string
|
||||
"""
|
||||
if mb < 1:
|
||||
return f"{mb * 1024:.1f} KB"
|
||||
elif mb < 1024:
|
||||
return f"{mb:.2f} MB"
|
||||
else:
|
||||
return f"{mb / 1024:.2f} GB"
|
||||
|
||||
|
||||
def calculate_pass_rate(passed: int, total: int) -> float:
|
||||
"""
|
||||
Calculate pass rate.
|
||||
|
||||
Args:
|
||||
passed: Number of passed evaluations
|
||||
total: Total number of evaluations
|
||||
|
||||
Returns:
|
||||
Pass rate as a float between 0 and 1
|
||||
"""
|
||||
if total == 0:
|
||||
return 0.0
|
||||
return passed / total
|
||||
@@ -1054,7 +1054,7 @@ class LLM:
|
||||
**kwargs
|
||||
) -> str:
|
||||
"""Enhanced get_response with all OpenAI-like features"""
|
||||
logging.info(f"Getting response from {self.model}")
|
||||
logging.debug(f"Getting response from {self.model}")
|
||||
# Log all self values when in debug mode
|
||||
self._log_llm_config(
|
||||
'LLM instance',
|
||||
@@ -2472,7 +2472,7 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
"""Async version of get_response with identical functionality."""
|
||||
try:
|
||||
import litellm
|
||||
logging.info(f"Getting async response from {self.model}")
|
||||
logging.debug(f"Getting async response from {self.model}")
|
||||
# Log all self values when in debug mode
|
||||
self._log_llm_config(
|
||||
'get_response_async',
|
||||
|
||||
@@ -19,6 +19,15 @@ from rich.console import Console
|
||||
from rich.live import Live
|
||||
import inspect
|
||||
|
||||
# Import display_tool_call for callback support (lazy import to avoid circular imports)
|
||||
_display_tool_call = None
|
||||
def _get_display_tool_call():
|
||||
global _display_tool_call
|
||||
if _display_tool_call is None:
|
||||
from ..main import display_tool_call
|
||||
_display_tool_call = display_tool_call
|
||||
return _display_tool_call
|
||||
|
||||
# Constants
|
||||
LOCAL_SERVER_API_KEY_PLACEHOLDER = "not-needed"
|
||||
|
||||
@@ -978,16 +987,19 @@ class OpenAIClient:
|
||||
function_name = tool_call.function.name
|
||||
arguments = json.loads(tool_call.function.arguments)
|
||||
|
||||
# Always trigger callback for tool call tracking (even when verbose=False)
|
||||
display_tool_call_fn = _get_display_tool_call()
|
||||
display_tool_call_fn(f"Calling function: {function_name}", console=console if verbose else None)
|
||||
|
||||
if verbose and console:
|
||||
console.print(f"[bold]Calling function:[/bold] {function_name}")
|
||||
console.print(f"[dim]Arguments:[/dim] {arguments}")
|
||||
|
||||
# Execute the tool
|
||||
tool_result = execute_tool_fn(function_name, arguments)
|
||||
results_str = json.dumps(tool_result) if tool_result else "Function returned an empty output"
|
||||
|
||||
if verbose and console:
|
||||
console.print(f"[dim]Result:[/dim] {results_str}")
|
||||
# Trigger callback with result
|
||||
display_tool_call_fn(f"Function {function_name} returned: {results_str[:200]}{'...' if len(results_str) > 200 else ''}", console=console if verbose else None)
|
||||
|
||||
messages.append({
|
||||
"role": "tool",
|
||||
@@ -1150,8 +1162,11 @@ class OpenAIClient:
|
||||
function_name = tool_call.function.name
|
||||
arguments = json.loads(tool_call.function.arguments)
|
||||
|
||||
# Always trigger callback for tool call tracking (even when verbose=False)
|
||||
display_tool_call_fn = _get_display_tool_call()
|
||||
display_tool_call_fn(f"Calling function: {function_name}", console=console if verbose else None)
|
||||
|
||||
if verbose and console:
|
||||
console.print(f"[bold]Calling function:[/bold] {function_name}")
|
||||
console.print(f"[dim]Arguments:[/dim] {arguments}")
|
||||
|
||||
# Execute the tool (async)
|
||||
@@ -1167,8 +1182,8 @@ class OpenAIClient:
|
||||
|
||||
results_str = json.dumps(tool_result) if tool_result else "Function returned an empty output"
|
||||
|
||||
if verbose and console:
|
||||
console.print(f"[dim]Result:[/dim] {results_str}")
|
||||
# Trigger callback with result
|
||||
display_tool_call_fn(f"Function {function_name} returned: {results_str[:200]}{'...' if len(results_str) > 200 else ''}", console=console if verbose else None)
|
||||
|
||||
messages.append({
|
||||
"role": "tool",
|
||||
@@ -1299,6 +1314,10 @@ class OpenAIClient:
|
||||
yield f"\n[Error parsing arguments for {function_name if 'function_name' in locals() else 'unknown function'}: {str(e)}]"
|
||||
continue
|
||||
|
||||
# Always trigger callback for tool call tracking (even when verbose=False)
|
||||
display_tool_call_fn = _get_display_tool_call()
|
||||
display_tool_call_fn(f"Calling function: {function_name}", console=None)
|
||||
|
||||
if verbose:
|
||||
yield f"\n[Calling function: {function_name}]"
|
||||
|
||||
@@ -1311,6 +1330,9 @@ class OpenAIClient:
|
||||
if verbose:
|
||||
yield f"\n[Function error: {str(e)}]"
|
||||
|
||||
# Trigger callback with result
|
||||
display_tool_call_fn(f"Function {function_name} returned: {results_str[:200]}{'...' if len(results_str) > 200 else ''}", console=None)
|
||||
|
||||
if verbose:
|
||||
yield f"\n[Function result: {results_str}]"
|
||||
|
||||
|
||||
@@ -231,15 +231,15 @@ def display_tool_call(message: str, console=None):
|
||||
if not message or not message.strip():
|
||||
logging.debug("Empty message in display_tool_call, returning early")
|
||||
return
|
||||
if console is None:
|
||||
console = Console()
|
||||
message = _clean_display_content(str(message))
|
||||
logging.debug(f"Cleaned message in display_tool_call: {repr(message)}")
|
||||
|
||||
# Execute synchronous callbacks
|
||||
# Execute synchronous callbacks (always, even when console is None)
|
||||
execute_sync_callback('tool_call', message=message)
|
||||
|
||||
console.print(Panel.fit(Text(message, style="bold cyan"), title="Tool Call", border_style="green"))
|
||||
# Only print panel if console is provided (verbose mode)
|
||||
if console is not None:
|
||||
console.print(Panel.fit(Text(message, style="bold cyan"), title="Tool Call", border_style="green"))
|
||||
|
||||
def display_error(message: str, console=None):
|
||||
if not message or not message.strip():
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Agent Skills module for PraisonAI Agents.
|
||||
|
||||
This module provides support for the open Agent Skills standard (agentskills.io),
|
||||
enabling agents to load and use modular capabilities through SKILL.md files.
|
||||
|
||||
Zero Performance Impact:
|
||||
- All imports are lazy loaded via __getattr__
|
||||
- Skills only loaded when explicitly enabled
|
||||
- No auto-discovery at import time
|
||||
|
||||
Usage:
|
||||
from praisonaiagents.skills import SkillManager, SkillProperties
|
||||
|
||||
manager = SkillManager()
|
||||
manager.discover(["./skills"])
|
||||
prompt_xml = manager.to_prompt()
|
||||
"""
|
||||
|
||||
__all__ = [
|
||||
# Models
|
||||
"SkillProperties",
|
||||
"SkillMetadata",
|
||||
"ParseError",
|
||||
"ValidationError",
|
||||
# Parser
|
||||
"parse_frontmatter",
|
||||
"find_skill_md",
|
||||
"read_properties",
|
||||
# Validator
|
||||
"validate",
|
||||
"validate_metadata",
|
||||
# Prompt
|
||||
"to_prompt",
|
||||
"generate_skills_xml",
|
||||
"format_skill_for_prompt",
|
||||
# Discovery
|
||||
"discover_skills",
|
||||
"get_default_skill_dirs",
|
||||
# Loader
|
||||
"SkillLoader",
|
||||
# Manager
|
||||
"SkillManager",
|
||||
]
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
"""Lazy load module components to avoid import overhead."""
|
||||
if name in ("SkillProperties", "SkillMetadata", "ParseError", "ValidationError"):
|
||||
from .models import SkillProperties, SkillMetadata, ParseError, ValidationError
|
||||
return locals()[name]
|
||||
|
||||
if name in ("parse_frontmatter", "find_skill_md", "read_properties"):
|
||||
from .parser import parse_frontmatter, find_skill_md, read_properties
|
||||
return locals()[name]
|
||||
|
||||
if name in ("validate", "validate_metadata", "_validate_name", "_validate_description", "_validate_compatibility"):
|
||||
from .validator import validate, validate_metadata, _validate_name, _validate_description, _validate_compatibility
|
||||
return locals()[name]
|
||||
|
||||
if name in ("to_prompt", "generate_skills_xml", "format_skill_for_prompt"):
|
||||
from .prompt import to_prompt, generate_skills_xml, format_skill_for_prompt
|
||||
return locals()[name]
|
||||
|
||||
if name in ("discover_skills", "get_default_skill_dirs"):
|
||||
from .discovery import discover_skills, get_default_skill_dirs
|
||||
return locals()[name]
|
||||
|
||||
if name == "SkillLoader":
|
||||
from .loader import SkillLoader
|
||||
return SkillLoader
|
||||
|
||||
if name == "SkillManager":
|
||||
from .manager import SkillManager
|
||||
return SkillManager
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Skill discovery and directory scanning."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
from .parser import find_skill_md, read_properties
|
||||
from .models import SkillProperties
|
||||
|
||||
|
||||
def get_default_skill_dirs() -> List[Path]:
|
||||
"""Get default skill directory locations.
|
||||
|
||||
Returns directories in precedence order (high to low):
|
||||
1. Project: ./.praison/skills/ or ./.claude/skills/
|
||||
2. User: ~/.praison/skills/
|
||||
3. System: /etc/praison/skills/ (admin-managed)
|
||||
|
||||
Returns:
|
||||
List of existing skill directories
|
||||
"""
|
||||
dirs = []
|
||||
cwd = Path.cwd()
|
||||
|
||||
# Project-level directories
|
||||
project_dirs = [
|
||||
cwd / ".praison" / "skills",
|
||||
cwd / ".claude" / "skills",
|
||||
]
|
||||
for d in project_dirs:
|
||||
if d.exists() and d.is_dir():
|
||||
dirs.append(d)
|
||||
|
||||
# User-level directory
|
||||
user_dir = Path.home() / ".praison" / "skills"
|
||||
if user_dir.exists() and user_dir.is_dir():
|
||||
dirs.append(user_dir)
|
||||
|
||||
# System-level directory (Unix-like systems)
|
||||
system_dir = Path("/etc/praison/skills")
|
||||
if system_dir.exists() and system_dir.is_dir():
|
||||
dirs.append(system_dir)
|
||||
|
||||
return dirs
|
||||
|
||||
|
||||
def discover_skills(
|
||||
skill_dirs: Optional[List[str]] = None,
|
||||
include_defaults: bool = True,
|
||||
) -> List[SkillProperties]:
|
||||
"""Discover all valid skills in the given directories.
|
||||
|
||||
Args:
|
||||
skill_dirs: List of directory paths to scan for skills.
|
||||
Each directory should contain skill subdirectories.
|
||||
include_defaults: Whether to include default skill directories
|
||||
|
||||
Returns:
|
||||
List of SkillProperties for all valid skills found
|
||||
"""
|
||||
all_dirs = []
|
||||
|
||||
# Add explicit directories
|
||||
if skill_dirs:
|
||||
for d in skill_dirs:
|
||||
path = Path(d).expanduser().resolve()
|
||||
if path.exists() and path.is_dir():
|
||||
all_dirs.append(path)
|
||||
|
||||
# Add default directories
|
||||
if include_defaults:
|
||||
all_dirs.extend(get_default_skill_dirs())
|
||||
|
||||
# Remove duplicates while preserving order
|
||||
seen = set()
|
||||
unique_dirs = []
|
||||
for d in all_dirs:
|
||||
if d not in seen:
|
||||
seen.add(d)
|
||||
unique_dirs.append(d)
|
||||
|
||||
skills = []
|
||||
|
||||
for parent_dir in unique_dirs:
|
||||
# Each subdirectory in parent_dir might be a skill
|
||||
try:
|
||||
for item in parent_dir.iterdir():
|
||||
if not item.is_dir():
|
||||
continue
|
||||
|
||||
# Check if this directory contains a SKILL.md
|
||||
skill_md = find_skill_md(item)
|
||||
if skill_md is None:
|
||||
continue
|
||||
|
||||
try:
|
||||
props = read_properties(item)
|
||||
skills.append(props)
|
||||
except Exception:
|
||||
# Skip invalid skills
|
||||
continue
|
||||
except PermissionError:
|
||||
# Skip directories we can't read
|
||||
continue
|
||||
|
||||
return skills
|
||||
|
||||
|
||||
def discover_skill(skill_path: str) -> Optional[SkillProperties]:
|
||||
"""Discover a single skill from a directory path.
|
||||
|
||||
Args:
|
||||
skill_path: Path to a skill directory
|
||||
|
||||
Returns:
|
||||
SkillProperties if valid, None otherwise
|
||||
"""
|
||||
path = Path(skill_path).expanduser().resolve()
|
||||
|
||||
if not path.exists() or not path.is_dir():
|
||||
return None
|
||||
|
||||
skill_md = find_skill_md(path)
|
||||
if skill_md is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
return read_properties(path)
|
||||
except Exception:
|
||||
return None
|
||||
@@ -0,0 +1,236 @@
|
||||
"""Progressive skill loading for Agent Skills."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .parser import find_skill_md, read_properties, parse_frontmatter
|
||||
from .models import SkillProperties, SkillMetadata
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoadedSkill:
|
||||
"""A fully or partially loaded skill.
|
||||
|
||||
Supports progressive disclosure:
|
||||
- Level 1: metadata only (name, description, location)
|
||||
- Level 2: full instructions (SKILL.md body)
|
||||
- Level 3: resources (scripts, references, assets)
|
||||
"""
|
||||
|
||||
properties: SkillProperties
|
||||
instructions: Optional[str] = None
|
||||
resources_loaded: bool = False
|
||||
_scripts: dict = field(default_factory=dict)
|
||||
_references: dict = field(default_factory=dict)
|
||||
_assets: dict = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def metadata(self) -> SkillMetadata:
|
||||
"""Get lightweight metadata for system prompt."""
|
||||
return SkillMetadata.from_properties(self.properties)
|
||||
|
||||
@property
|
||||
def is_activated(self) -> bool:
|
||||
"""Check if skill has been activated (instructions loaded)."""
|
||||
return self.instructions is not None
|
||||
|
||||
def get_scripts(self) -> dict:
|
||||
"""Get loaded scripts."""
|
||||
return self._scripts
|
||||
|
||||
def get_references(self) -> dict:
|
||||
"""Get loaded references."""
|
||||
return self._references
|
||||
|
||||
def get_assets(self) -> dict:
|
||||
"""Get loaded assets."""
|
||||
return self._assets
|
||||
|
||||
|
||||
class SkillLoader:
|
||||
"""Loader for Agent Skills with progressive disclosure support.
|
||||
|
||||
Usage:
|
||||
loader = SkillLoader()
|
||||
|
||||
# Level 1: Load metadata only
|
||||
skill = loader.load_metadata("/path/to/skill")
|
||||
|
||||
# Level 2: Activate skill (load instructions)
|
||||
loader.activate(skill)
|
||||
|
||||
# Level 3: Load specific resources
|
||||
loader.load_scripts(skill)
|
||||
loader.load_references(skill)
|
||||
"""
|
||||
|
||||
def load_metadata(self, skill_path: str) -> Optional[LoadedSkill]:
|
||||
"""Load skill metadata only (Level 1).
|
||||
|
||||
This is the minimal load for system prompt injection.
|
||||
Only reads the YAML frontmatter, not the full SKILL.md body.
|
||||
|
||||
Args:
|
||||
skill_path: Path to skill directory
|
||||
|
||||
Returns:
|
||||
LoadedSkill with metadata only, or None if invalid
|
||||
"""
|
||||
path = Path(skill_path).expanduser().resolve()
|
||||
|
||||
if not path.exists() or not path.is_dir():
|
||||
return None
|
||||
|
||||
try:
|
||||
props = read_properties(path)
|
||||
return LoadedSkill(properties=props)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def activate(self, skill: LoadedSkill) -> bool:
|
||||
"""Activate a skill by loading its full instructions (Level 2).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance to activate
|
||||
|
||||
Returns:
|
||||
True if activation succeeded
|
||||
"""
|
||||
if skill.is_activated:
|
||||
return True
|
||||
|
||||
if skill.properties.path is None:
|
||||
return False
|
||||
|
||||
skill_md = find_skill_md(skill.properties.path)
|
||||
if skill_md is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
content = skill_md.read_text()
|
||||
_, body = parse_frontmatter(content)
|
||||
skill.instructions = body
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def load_scripts(self, skill: LoadedSkill) -> dict:
|
||||
"""Load scripts from skill's scripts/ directory (Level 3).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance
|
||||
|
||||
Returns:
|
||||
Dict mapping script names to their contents
|
||||
"""
|
||||
if skill.properties.path is None:
|
||||
return {}
|
||||
|
||||
scripts_dir = skill.properties.path / "scripts"
|
||||
if not scripts_dir.exists() or not scripts_dir.is_dir():
|
||||
return {}
|
||||
|
||||
scripts = {}
|
||||
try:
|
||||
for script_file in scripts_dir.iterdir():
|
||||
if script_file.is_file():
|
||||
try:
|
||||
scripts[script_file.name] = script_file.read_text()
|
||||
except Exception:
|
||||
continue
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
skill._scripts = scripts
|
||||
return scripts
|
||||
|
||||
def load_references(self, skill: LoadedSkill) -> dict:
|
||||
"""Load references from skill's references/ directory (Level 3).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance
|
||||
|
||||
Returns:
|
||||
Dict mapping reference names to their contents
|
||||
"""
|
||||
if skill.properties.path is None:
|
||||
return {}
|
||||
|
||||
refs_dir = skill.properties.path / "references"
|
||||
if not refs_dir.exists() or not refs_dir.is_dir():
|
||||
return {}
|
||||
|
||||
refs = {}
|
||||
try:
|
||||
for ref_file in refs_dir.iterdir():
|
||||
if ref_file.is_file():
|
||||
try:
|
||||
refs[ref_file.name] = ref_file.read_text()
|
||||
except Exception:
|
||||
continue
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
skill._references = refs
|
||||
return refs
|
||||
|
||||
def load_assets(self, skill: LoadedSkill) -> dict:
|
||||
"""Load asset paths from skill's assets/ directory (Level 3).
|
||||
|
||||
Note: For binary assets, only paths are returned, not contents.
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance
|
||||
|
||||
Returns:
|
||||
Dict mapping asset names to their paths
|
||||
"""
|
||||
if skill.properties.path is None:
|
||||
return {}
|
||||
|
||||
assets_dir = skill.properties.path / "assets"
|
||||
if not assets_dir.exists() or not assets_dir.is_dir():
|
||||
return {}
|
||||
|
||||
assets = {}
|
||||
try:
|
||||
for asset_file in assets_dir.iterdir():
|
||||
if asset_file.is_file():
|
||||
assets[asset_file.name] = str(asset_file)
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
skill._assets = assets
|
||||
skill.resources_loaded = True
|
||||
return assets
|
||||
|
||||
def load_all_resources(self, skill: LoadedSkill) -> None:
|
||||
"""Load all resources for a skill (Level 3 complete).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill instance
|
||||
"""
|
||||
self.load_scripts(skill)
|
||||
self.load_references(skill)
|
||||
self.load_assets(skill)
|
||||
skill.resources_loaded = True
|
||||
|
||||
@classmethod
|
||||
def load(cls, skill_path: str, activate: bool = False) -> Optional[LoadedSkill]:
|
||||
"""Convenience method to load a skill.
|
||||
|
||||
Args:
|
||||
skill_path: Path to skill directory
|
||||
activate: Whether to also load instructions
|
||||
|
||||
Returns:
|
||||
LoadedSkill or None if invalid
|
||||
"""
|
||||
loader = cls()
|
||||
skill = loader.load_metadata(skill_path)
|
||||
|
||||
if skill and activate:
|
||||
loader.activate(skill)
|
||||
|
||||
return skill
|
||||
@@ -0,0 +1,189 @@
|
||||
"""SkillManager for Agent Skills integration."""
|
||||
|
||||
from typing import List, Optional, Dict
|
||||
|
||||
from .models import SkillMetadata
|
||||
from .discovery import discover_skills
|
||||
from .loader import SkillLoader, LoadedSkill
|
||||
from .prompt import generate_skills_xml
|
||||
|
||||
|
||||
class SkillManager:
|
||||
"""Manager for discovering, loading, and using Agent Skills.
|
||||
|
||||
This is the main entry point for Agent Skills integration.
|
||||
Supports progressive disclosure and zero performance impact
|
||||
when skills are not used.
|
||||
|
||||
Usage:
|
||||
manager = SkillManager()
|
||||
|
||||
# Discover skills from directories
|
||||
manager.discover(["./skills", "~/.praison/skills"])
|
||||
|
||||
# Get XML for system prompt
|
||||
prompt_xml = manager.to_prompt()
|
||||
|
||||
# Activate a specific skill
|
||||
skill = manager.get_skill("pdf-processing")
|
||||
manager.activate(skill)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the SkillManager."""
|
||||
self._skills: Dict[str, LoadedSkill] = {}
|
||||
self._loader = SkillLoader()
|
||||
self._discovered = False
|
||||
|
||||
@property
|
||||
def skills(self) -> List[LoadedSkill]:
|
||||
"""Get all loaded skills."""
|
||||
return list(self._skills.values())
|
||||
|
||||
@property
|
||||
def skill_names(self) -> List[str]:
|
||||
"""Get names of all loaded skills."""
|
||||
return list(self._skills.keys())
|
||||
|
||||
def discover(
|
||||
self,
|
||||
skill_dirs: Optional[List[str]] = None,
|
||||
include_defaults: bool = True,
|
||||
) -> int:
|
||||
"""Discover skills from directories.
|
||||
|
||||
Args:
|
||||
skill_dirs: List of directory paths to scan
|
||||
include_defaults: Whether to include default skill directories
|
||||
|
||||
Returns:
|
||||
Number of skills discovered
|
||||
"""
|
||||
props_list = discover_skills(skill_dirs, include_defaults)
|
||||
|
||||
for props in props_list:
|
||||
if props.name not in self._skills:
|
||||
self._skills[props.name] = LoadedSkill(properties=props)
|
||||
|
||||
self._discovered = True
|
||||
return len(props_list)
|
||||
|
||||
def add_skill(self, skill_path: str) -> Optional[LoadedSkill]:
|
||||
"""Add a single skill from a directory path.
|
||||
|
||||
Args:
|
||||
skill_path: Path to skill directory
|
||||
|
||||
Returns:
|
||||
LoadedSkill if successful, None otherwise
|
||||
"""
|
||||
skill = self._loader.load_metadata(skill_path)
|
||||
if skill:
|
||||
self._skills[skill.properties.name] = skill
|
||||
return skill
|
||||
|
||||
def get_skill(self, name: str) -> Optional[LoadedSkill]:
|
||||
"""Get a skill by name.
|
||||
|
||||
Args:
|
||||
name: Skill name
|
||||
|
||||
Returns:
|
||||
LoadedSkill or None if not found
|
||||
"""
|
||||
return self._skills.get(name)
|
||||
|
||||
def activate(self, skill: LoadedSkill) -> bool:
|
||||
"""Activate a skill (load its instructions).
|
||||
|
||||
Args:
|
||||
skill: LoadedSkill to activate
|
||||
|
||||
Returns:
|
||||
True if activation succeeded
|
||||
"""
|
||||
return self._loader.activate(skill)
|
||||
|
||||
def activate_by_name(self, name: str) -> bool:
|
||||
"""Activate a skill by name.
|
||||
|
||||
Args:
|
||||
name: Skill name
|
||||
|
||||
Returns:
|
||||
True if activation succeeded
|
||||
"""
|
||||
skill = self.get_skill(name)
|
||||
if skill is None:
|
||||
return False
|
||||
return self.activate(skill)
|
||||
|
||||
def get_available_skills(self) -> List[SkillMetadata]:
|
||||
"""Get metadata for all available skills.
|
||||
|
||||
This is used for system prompt injection.
|
||||
|
||||
Returns:
|
||||
List of SkillMetadata instances
|
||||
"""
|
||||
return [skill.metadata for skill in self._skills.values()]
|
||||
|
||||
def to_prompt(self) -> str:
|
||||
"""Generate XML prompt for available skills.
|
||||
|
||||
Returns:
|
||||
XML string with <available_skills> block
|
||||
"""
|
||||
metadata_list = self.get_available_skills()
|
||||
return generate_skills_xml(metadata_list)
|
||||
|
||||
def get_instructions(self, name: str) -> Optional[str]:
|
||||
"""Get instructions for a skill, activating if needed.
|
||||
|
||||
Args:
|
||||
name: Skill name
|
||||
|
||||
Returns:
|
||||
Skill instructions or None if not found
|
||||
"""
|
||||
skill = self.get_skill(name)
|
||||
if skill is None:
|
||||
return None
|
||||
|
||||
if not skill.is_activated:
|
||||
self.activate(skill)
|
||||
|
||||
return skill.instructions
|
||||
|
||||
def load_resources(self, name: str) -> bool:
|
||||
"""Load all resources for a skill.
|
||||
|
||||
Args:
|
||||
name: Skill name
|
||||
|
||||
Returns:
|
||||
True if resources loaded successfully
|
||||
"""
|
||||
skill = self.get_skill(name)
|
||||
if skill is None:
|
||||
return False
|
||||
|
||||
self._loader.load_all_resources(skill)
|
||||
return True
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear all loaded skills."""
|
||||
self._skills.clear()
|
||||
self._discovered = False
|
||||
|
||||
def __len__(self) -> int:
|
||||
"""Get number of loaded skills."""
|
||||
return len(self._skills)
|
||||
|
||||
def __contains__(self, name: str) -> bool:
|
||||
"""Check if a skill is loaded."""
|
||||
return name in self._skills
|
||||
|
||||
def __iter__(self):
|
||||
"""Iterate over loaded skills."""
|
||||
return iter(self._skills.values())
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Data models for Agent Skills."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class ParseError(Exception):
|
||||
"""Raised when SKILL.md parsing fails."""
|
||||
pass
|
||||
|
||||
|
||||
class ValidationError(Exception):
|
||||
"""Raised when skill validation fails."""
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class SkillProperties:
|
||||
"""Properties parsed from a skill's SKILL.md frontmatter.
|
||||
|
||||
Attributes:
|
||||
name: Skill name in kebab-case (required)
|
||||
description: What the skill does and when the model should use it (required)
|
||||
license: License for the skill (optional)
|
||||
compatibility: Compatibility information for the skill (optional)
|
||||
allowed_tools: Tool patterns the skill requires (optional, experimental)
|
||||
metadata: Key-value pairs for client-specific properties (defaults to
|
||||
empty dict; omitted from to_dict() output when empty)
|
||||
path: Path to the skill directory (optional, added for PraisonAI)
|
||||
"""
|
||||
|
||||
name: str
|
||||
description: str
|
||||
license: Optional[str] = None
|
||||
compatibility: Optional[str] = None
|
||||
allowed_tools: Optional[str] = None
|
||||
metadata: dict = field(default_factory=dict)
|
||||
path: Optional[Path] = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Convert to dictionary, excluding None values and empty metadata."""
|
||||
result = {"name": self.name, "description": self.description}
|
||||
if self.license is not None:
|
||||
result["license"] = self.license
|
||||
if self.compatibility is not None:
|
||||
result["compatibility"] = self.compatibility
|
||||
if self.allowed_tools is not None:
|
||||
result["allowed-tools"] = self.allowed_tools
|
||||
if self.metadata:
|
||||
result["metadata"] = self.metadata
|
||||
return result
|
||||
|
||||
|
||||
@dataclass
|
||||
class SkillMetadata:
|
||||
"""Lightweight skill metadata for system prompt injection.
|
||||
|
||||
This is the minimal information needed to include a skill in the
|
||||
system prompt's <available_skills> block (~50-100 tokens per skill).
|
||||
|
||||
Attributes:
|
||||
name: Skill name
|
||||
description: What the skill does and when to use it
|
||||
location: Absolute path to the SKILL.md file
|
||||
"""
|
||||
|
||||
name: str
|
||||
description: str
|
||||
location: str
|
||||
|
||||
@classmethod
|
||||
def from_properties(cls, props: SkillProperties) -> "SkillMetadata":
|
||||
"""Create SkillMetadata from SkillProperties.
|
||||
|
||||
Args:
|
||||
props: SkillProperties instance with path set
|
||||
|
||||
Returns:
|
||||
SkillMetadata with location pointing to SKILL.md
|
||||
"""
|
||||
if props.path:
|
||||
# Look for SKILL.md or skill.md
|
||||
skill_md = props.path / "SKILL.md"
|
||||
if not skill_md.exists():
|
||||
skill_md = props.path / "skill.md"
|
||||
location = str(skill_md)
|
||||
else:
|
||||
location = ""
|
||||
|
||||
return cls(
|
||||
name=props.name,
|
||||
description=props.description,
|
||||
location=location
|
||||
)
|
||||
@@ -0,0 +1,165 @@
|
||||
"""YAML frontmatter parsing for SKILL.md files."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Optional, Tuple
|
||||
import re
|
||||
|
||||
from .models import ParseError, ValidationError, SkillProperties
|
||||
|
||||
|
||||
def find_skill_md(skill_dir: Path) -> Optional[Path]:
|
||||
"""Find the SKILL.md file in a skill directory.
|
||||
|
||||
Prefers SKILL.md (uppercase) but accepts skill.md (lowercase).
|
||||
|
||||
Args:
|
||||
skill_dir: Path to the skill directory
|
||||
|
||||
Returns:
|
||||
Path to the SKILL.md file, or None if not found
|
||||
"""
|
||||
for name in ("SKILL.md", "skill.md"):
|
||||
path = skill_dir / name
|
||||
if path.exists():
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
def parse_frontmatter(content: str) -> Tuple[dict, str]:
|
||||
"""Parse YAML frontmatter from SKILL.md content.
|
||||
|
||||
Args:
|
||||
content: Raw content of SKILL.md file
|
||||
|
||||
Returns:
|
||||
Tuple of (metadata dict, markdown body)
|
||||
|
||||
Raises:
|
||||
ParseError: If frontmatter is missing or invalid
|
||||
"""
|
||||
if not content.startswith("---"):
|
||||
raise ParseError("SKILL.md must start with YAML frontmatter (---)")
|
||||
|
||||
parts = content.split("---", 2)
|
||||
if len(parts) < 3:
|
||||
raise ParseError("SKILL.md frontmatter not properly closed with ---")
|
||||
|
||||
frontmatter_str = parts[1]
|
||||
body = parts[2].strip()
|
||||
|
||||
try:
|
||||
metadata = _parse_yaml(frontmatter_str)
|
||||
except Exception as e:
|
||||
raise ParseError(f"Invalid YAML in frontmatter: {e}")
|
||||
|
||||
if not isinstance(metadata, dict):
|
||||
raise ParseError("SKILL.md frontmatter must be a YAML mapping")
|
||||
|
||||
# Convert metadata values to strings where needed
|
||||
if "metadata" in metadata and isinstance(metadata["metadata"], dict):
|
||||
metadata["metadata"] = {str(k): str(v) for k, v in metadata["metadata"].items()}
|
||||
|
||||
return metadata, body
|
||||
|
||||
|
||||
def _parse_yaml(yaml_str: str) -> dict:
|
||||
"""Parse YAML string to dictionary.
|
||||
|
||||
Uses a simple parser to avoid strictyaml dependency.
|
||||
Falls back to yaml if available.
|
||||
|
||||
Args:
|
||||
yaml_str: YAML string to parse
|
||||
|
||||
Returns:
|
||||
Parsed dictionary
|
||||
"""
|
||||
# Try to use PyYAML if available
|
||||
try:
|
||||
import yaml
|
||||
return yaml.safe_load(yaml_str) or {}
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Simple fallback parser for basic YAML
|
||||
result = {}
|
||||
current_key = None
|
||||
current_dict = None
|
||||
|
||||
for line in yaml_str.strip().split("\n"):
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
# Check for nested dict
|
||||
if line.startswith(" ") and current_dict is not None:
|
||||
match = re.match(r"^\s+(\w+):\s*(.*)$", line)
|
||||
if match:
|
||||
key, value = match.groups()
|
||||
current_dict[key] = value.strip().strip('"').strip("'")
|
||||
continue
|
||||
|
||||
# Top-level key
|
||||
match = re.match(r"^(\S+):\s*(.*)$", line)
|
||||
if match:
|
||||
key, value = match.groups()
|
||||
value = value.strip()
|
||||
|
||||
if not value:
|
||||
# Could be start of nested dict
|
||||
result[key] = {}
|
||||
current_key = key
|
||||
current_dict = result[key]
|
||||
else:
|
||||
result[key] = value.strip('"').strip("'")
|
||||
current_dict = None
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def read_properties(skill_dir: Path) -> SkillProperties:
|
||||
"""Read skill properties from SKILL.md frontmatter.
|
||||
|
||||
This function parses the frontmatter and returns properties.
|
||||
It does NOT perform full validation. Use validate() for that.
|
||||
|
||||
Args:
|
||||
skill_dir: Path to the skill directory
|
||||
|
||||
Returns:
|
||||
SkillProperties with parsed metadata
|
||||
|
||||
Raises:
|
||||
ParseError: If SKILL.md is missing or has invalid YAML
|
||||
ValidationError: If required fields (name, description) are missing
|
||||
"""
|
||||
skill_dir = Path(skill_dir)
|
||||
skill_md = find_skill_md(skill_dir)
|
||||
|
||||
if skill_md is None:
|
||||
raise ParseError(f"SKILL.md not found in {skill_dir}")
|
||||
|
||||
content = skill_md.read_text()
|
||||
metadata, _ = parse_frontmatter(content)
|
||||
|
||||
if "name" not in metadata:
|
||||
raise ValidationError("Missing required field in frontmatter: name")
|
||||
if "description" not in metadata:
|
||||
raise ValidationError("Missing required field in frontmatter: description")
|
||||
|
||||
name = metadata["name"]
|
||||
description = metadata["description"]
|
||||
|
||||
if not isinstance(name, str) or not name.strip():
|
||||
raise ValidationError("Field 'name' must be a non-empty string")
|
||||
if not isinstance(description, str) or not description.strip():
|
||||
raise ValidationError("Field 'description' must be a non-empty string")
|
||||
|
||||
return SkillProperties(
|
||||
name=name.strip(),
|
||||
description=description.strip(),
|
||||
license=metadata.get("license"),
|
||||
compatibility=metadata.get("compatibility"),
|
||||
allowed_tools=metadata.get("allowed-tools"),
|
||||
metadata=metadata.get("metadata") or {},
|
||||
path=skill_dir,
|
||||
)
|
||||
@@ -0,0 +1,74 @@
|
||||
"""XML prompt generation for Agent Skills."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
import html
|
||||
|
||||
from .models import SkillMetadata
|
||||
from .parser import read_properties
|
||||
|
||||
|
||||
def format_skill_for_prompt(skill: SkillMetadata) -> str:
|
||||
"""Format a single skill as XML for system prompt.
|
||||
|
||||
Args:
|
||||
skill: SkillMetadata instance
|
||||
|
||||
Returns:
|
||||
XML string for the skill
|
||||
"""
|
||||
name = html.escape(skill.name)
|
||||
description = html.escape(skill.description)
|
||||
location = html.escape(skill.location)
|
||||
|
||||
return f""" <skill>
|
||||
<name>{name}</name>
|
||||
<description>{description}</description>
|
||||
<location>{location}</location>
|
||||
</skill>"""
|
||||
|
||||
|
||||
def generate_skills_xml(skills: List[SkillMetadata]) -> str:
|
||||
"""Generate XML block for available skills.
|
||||
|
||||
Args:
|
||||
skills: List of SkillMetadata instances
|
||||
|
||||
Returns:
|
||||
XML string with <available_skills> block
|
||||
"""
|
||||
if not skills:
|
||||
return "<available_skills>\n</available_skills>"
|
||||
|
||||
skill_entries = "\n".join(format_skill_for_prompt(s) for s in skills)
|
||||
return f"<available_skills>\n{skill_entries}\n</available_skills>"
|
||||
|
||||
|
||||
def to_prompt(skill_dirs: List[Path]) -> str:
|
||||
"""Generate prompt XML from skill directories.
|
||||
|
||||
This is the main entry point for generating the <available_skills>
|
||||
block to inject into agent system prompts.
|
||||
|
||||
Args:
|
||||
skill_dirs: List of paths to skill directories
|
||||
|
||||
Returns:
|
||||
XML string with <available_skills> block
|
||||
"""
|
||||
skills = []
|
||||
|
||||
for skill_dir in skill_dirs:
|
||||
skill_dir = Path(skill_dir)
|
||||
if not skill_dir.exists() or not skill_dir.is_dir():
|
||||
continue
|
||||
|
||||
try:
|
||||
props = read_properties(skill_dir)
|
||||
meta = SkillMetadata.from_properties(props)
|
||||
skills.append(meta)
|
||||
except Exception:
|
||||
# Skip invalid skills silently
|
||||
continue
|
||||
|
||||
return generate_skills_xml(skills)
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Skill validation logic."""
|
||||
|
||||
import unicodedata
|
||||
from pathlib import Path
|
||||
from typing import Optional, List
|
||||
|
||||
from .models import ParseError
|
||||
from .parser import find_skill_md, parse_frontmatter
|
||||
|
||||
MAX_SKILL_NAME_LENGTH = 64
|
||||
MAX_DESCRIPTION_LENGTH = 1024
|
||||
MAX_COMPATIBILITY_LENGTH = 500
|
||||
|
||||
ALLOWED_FIELDS = {
|
||||
"name",
|
||||
"description",
|
||||
"license",
|
||||
"allowed-tools",
|
||||
"metadata",
|
||||
"compatibility",
|
||||
}
|
||||
|
||||
|
||||
def _validate_name(name: str, skill_dir: Optional[Path]) -> List[str]:
|
||||
"""Validate skill name format and directory match.
|
||||
|
||||
Skill names support i18n characters (Unicode letters) plus hyphens.
|
||||
Names must be lowercase and cannot start/end with hyphens.
|
||||
"""
|
||||
errors = []
|
||||
|
||||
if not name or not isinstance(name, str) or not name.strip():
|
||||
errors.append("Field 'name' must be a non-empty string")
|
||||
return errors
|
||||
|
||||
name = unicodedata.normalize("NFKC", name.strip())
|
||||
|
||||
if len(name) > MAX_SKILL_NAME_LENGTH:
|
||||
errors.append(
|
||||
f"Skill name '{name}' exceeds {MAX_SKILL_NAME_LENGTH} character limit "
|
||||
f"({len(name)} chars)"
|
||||
)
|
||||
|
||||
if name != name.lower():
|
||||
errors.append(f"Skill name '{name}' must be lowercase")
|
||||
|
||||
if name.startswith("-") or name.endswith("-"):
|
||||
errors.append("Skill name cannot start or end with a hyphen")
|
||||
|
||||
if "--" in name:
|
||||
errors.append("Skill name cannot contain consecutive hyphens")
|
||||
|
||||
if not all(c.isalnum() or c == "-" for c in name):
|
||||
errors.append(
|
||||
f"Skill name '{name}' contains invalid characters. "
|
||||
"Only letters, digits, and hyphens are allowed."
|
||||
)
|
||||
|
||||
if skill_dir:
|
||||
dir_name = unicodedata.normalize("NFKC", skill_dir.name)
|
||||
if dir_name != name:
|
||||
errors.append(
|
||||
f"Directory name '{skill_dir.name}' must match skill name '{name}'"
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def _validate_description(description: str) -> List[str]:
|
||||
"""Validate description format."""
|
||||
errors = []
|
||||
|
||||
if not description or not isinstance(description, str) or not description.strip():
|
||||
errors.append("Field 'description' must be a non-empty string")
|
||||
return errors
|
||||
|
||||
if len(description) > MAX_DESCRIPTION_LENGTH:
|
||||
errors.append(
|
||||
f"Description exceeds {MAX_DESCRIPTION_LENGTH} character limit "
|
||||
f"({len(description)} chars)"
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def _validate_compatibility(compatibility: str) -> List[str]:
|
||||
"""Validate compatibility format."""
|
||||
errors = []
|
||||
|
||||
if not isinstance(compatibility, str):
|
||||
errors.append("Field 'compatibility' must be a string")
|
||||
return errors
|
||||
|
||||
if len(compatibility) > MAX_COMPATIBILITY_LENGTH:
|
||||
errors.append(
|
||||
f"Compatibility exceeds {MAX_COMPATIBILITY_LENGTH} character limit "
|
||||
f"({len(compatibility)} chars)"
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def _validate_metadata_fields(metadata: dict) -> List[str]:
|
||||
"""Validate that only allowed fields are present."""
|
||||
errors = []
|
||||
|
||||
extra_fields = set(metadata.keys()) - ALLOWED_FIELDS
|
||||
if extra_fields:
|
||||
errors.append(
|
||||
f"Unexpected fields in frontmatter: {', '.join(sorted(extra_fields))}. "
|
||||
f"Only {sorted(ALLOWED_FIELDS)} are allowed."
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def validate_metadata(metadata: dict, skill_dir: Optional[Path] = None) -> List[str]:
|
||||
"""Validate parsed skill metadata.
|
||||
|
||||
This is the core validation function that works on already-parsed metadata,
|
||||
avoiding duplicate file I/O when called from the parser.
|
||||
|
||||
Args:
|
||||
metadata: Parsed YAML frontmatter dictionary
|
||||
skill_dir: Optional path to skill directory (for name-directory match check)
|
||||
|
||||
Returns:
|
||||
List of validation error messages. Empty list means valid.
|
||||
"""
|
||||
errors = []
|
||||
errors.extend(_validate_metadata_fields(metadata))
|
||||
|
||||
if "name" not in metadata:
|
||||
errors.append("Missing required field in frontmatter: name")
|
||||
else:
|
||||
errors.extend(_validate_name(metadata["name"], skill_dir))
|
||||
|
||||
if "description" not in metadata:
|
||||
errors.append("Missing required field in frontmatter: description")
|
||||
else:
|
||||
errors.extend(_validate_description(metadata["description"]))
|
||||
|
||||
if "compatibility" in metadata:
|
||||
errors.extend(_validate_compatibility(metadata["compatibility"]))
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def validate(skill_dir: Path) -> List[str]:
|
||||
"""Validate a skill directory.
|
||||
|
||||
Args:
|
||||
skill_dir: Path to the skill directory
|
||||
|
||||
Returns:
|
||||
List of validation error messages. Empty list means valid.
|
||||
"""
|
||||
skill_dir = Path(skill_dir)
|
||||
|
||||
if not skill_dir.exists():
|
||||
return [f"Path does not exist: {skill_dir}"]
|
||||
|
||||
if not skill_dir.is_dir():
|
||||
return [f"Not a directory: {skill_dir}"]
|
||||
|
||||
skill_md = find_skill_md(skill_dir)
|
||||
if skill_md is None:
|
||||
return ["Missing required file: SKILL.md"]
|
||||
|
||||
try:
|
||||
content = skill_md.read_text()
|
||||
metadata, _ = parse_frontmatter(content)
|
||||
except ParseError as e:
|
||||
return [str(e)]
|
||||
|
||||
return validate_metadata(metadata, skill_dir)
|
||||
@@ -222,6 +222,14 @@ TOOL_MAPPINGS = {
|
||||
# Unified Web Search (auto-fallback across providers)
|
||||
'search_web': ('.web_search', None),
|
||||
'get_available_providers': ('.web_search', None),
|
||||
|
||||
# Skill Tools (for Agent Skills script execution)
|
||||
'run_skill_script': ('.skill_tools', None),
|
||||
'read_skill_file': ('.skill_tools', None),
|
||||
'list_skill_scripts': ('.skill_tools', None),
|
||||
'create_skill_tools': ('.skill_tools', None),
|
||||
'SkillTools': ('.skill_tools', 'SkillTools'),
|
||||
'skill_tools': ('.skill_tools', None),
|
||||
}
|
||||
|
||||
_instances = {} # Cache for class instances
|
||||
@@ -259,11 +267,12 @@ def __getattr__(name: str) -> Any:
|
||||
'exa_search_async', 'exa_search_contents_async', 'exa_answer_async',
|
||||
'crawl4ai', 'crawl4ai_many', 'crawl4ai_extract', 'crawl4ai_llm_extract',
|
||||
'crawl4ai_sync', 'crawl4ai_extract_sync',
|
||||
'search_web', 'get_available_providers'
|
||||
'search_web', 'get_available_providers',
|
||||
'run_skill_script', 'read_skill_file', 'list_skill_scripts', 'create_skill_tools'
|
||||
]:
|
||||
return getattr(module, name)
|
||||
if name in ['file_tools', 'pandas_tools', 'wikipedia_tools',
|
||||
'newspaper_tools', 'arxiv_tools', 'spider_tools', 'duckdb_tools', 'mongodb_tools', 'csv_tools', 'json_tools', 'excel_tools', 'xml_tools', 'yaml_tools', 'calculator_tools', 'python_tools', 'shell_tools', 'cot_tools', 'tavily_tools', 'youdotcom_tools', 'exa_tools', 'crawl4ai_tools']:
|
||||
'newspaper_tools', 'arxiv_tools', 'spider_tools', 'duckdb_tools', 'mongodb_tools', 'csv_tools', 'json_tools', 'excel_tools', 'xml_tools', 'yaml_tools', 'calculator_tools', 'python_tools', 'shell_tools', 'cot_tools', 'tavily_tools', 'youdotcom_tools', 'exa_tools', 'crawl4ai_tools', 'skill_tools']:
|
||||
return module # Returns the callable module
|
||||
return getattr(module, name)
|
||||
else:
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
"""Tools for executing Agent Skills scripts.
|
||||
|
||||
This module provides tools for running scripts bundled with Agent Skills.
|
||||
Skills are modular packages that extend agent capabilities through SKILL.md
|
||||
files and optional scripts.
|
||||
|
||||
Usage:
|
||||
from praisonaiagents.tools import run_skill_script
|
||||
result = run_skill_script("./my-skill/scripts/script.py", "./data.csv")
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import os
|
||||
import logging
|
||||
|
||||
|
||||
class SkillTools:
|
||||
"""Tools for executing Agent Skills scripts."""
|
||||
|
||||
def __init__(self, working_directory: str = None):
|
||||
"""Initialize SkillTools.
|
||||
|
||||
Args:
|
||||
working_directory: Base directory for resolving relative paths.
|
||||
Defaults to current working directory.
|
||||
"""
|
||||
self._working_directory = working_directory or os.getcwd()
|
||||
|
||||
@property
|
||||
def working_directory(self) -> str:
|
||||
"""Get the working directory for path resolution."""
|
||||
return self._working_directory
|
||||
|
||||
@working_directory.setter
|
||||
def working_directory(self, value: str):
|
||||
"""Set the working directory for path resolution."""
|
||||
self._working_directory = value
|
||||
|
||||
def run_skill_script(
|
||||
self,
|
||||
script_path: str,
|
||||
args: str = "",
|
||||
timeout: int = 60
|
||||
) -> str:
|
||||
"""
|
||||
Execute a skill script from a skill's scripts/ directory.
|
||||
|
||||
Args:
|
||||
script_path: Path to the script file (e.g., ./csv-analyzer/scripts/script.py)
|
||||
args: Arguments to pass to the script (e.g., file path to analyze)
|
||||
timeout: Maximum execution time in seconds (default: 60)
|
||||
|
||||
Returns:
|
||||
The output from running the script, or error message if failed
|
||||
"""
|
||||
try:
|
||||
# Resolve the script path
|
||||
script_path = os.path.expanduser(script_path)
|
||||
if not os.path.isabs(script_path):
|
||||
script_path = os.path.join(self._working_directory, script_path)
|
||||
script_path = os.path.abspath(script_path)
|
||||
|
||||
if not os.path.exists(script_path):
|
||||
return f"Error: Script not found at {script_path}"
|
||||
|
||||
# Resolve args paths relative to working directory
|
||||
resolved_args = []
|
||||
if args:
|
||||
for arg in args.split():
|
||||
# If arg looks like a file path, resolve it
|
||||
if arg.startswith('./') or arg.startswith('../') or (
|
||||
not arg.startswith('-') and '/' in arg or '.' in arg
|
||||
):
|
||||
potential_path = os.path.join(self._working_directory, arg)
|
||||
if os.path.exists(potential_path):
|
||||
resolved_args.append(os.path.abspath(potential_path))
|
||||
else:
|
||||
# Keep original arg if path doesn't exist
|
||||
resolved_args.append(arg)
|
||||
else:
|
||||
resolved_args.append(arg)
|
||||
|
||||
# Determine how to run the script based on extension
|
||||
ext = os.path.splitext(script_path)[1].lower()
|
||||
|
||||
if ext == '.py':
|
||||
cmd = ['python', script_path] + resolved_args
|
||||
elif ext == '.sh':
|
||||
cmd = ['bash', script_path] + resolved_args
|
||||
elif ext == '.js':
|
||||
cmd = ['node', script_path] + resolved_args
|
||||
else:
|
||||
# Try to run directly
|
||||
cmd = [script_path] + resolved_args
|
||||
|
||||
logging.debug(f"Running skill script: {' '.join(cmd)}")
|
||||
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
cwd=self._working_directory
|
||||
)
|
||||
|
||||
output = result.stdout
|
||||
if result.stderr:
|
||||
output += f"\nStderr: {result.stderr}"
|
||||
if result.returncode != 0:
|
||||
output += f"\nExit code: {result.returncode}"
|
||||
|
||||
return output if output else "Script executed successfully (no output)"
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
return f"Error: Script execution timed out after {timeout} seconds"
|
||||
except FileNotFoundError as e:
|
||||
return f"Error: Could not find interpreter or script: {str(e)}"
|
||||
except PermissionError as e:
|
||||
return f"Error: Permission denied: {str(e)}"
|
||||
except Exception as e:
|
||||
return f"Error executing script: {str(e)}"
|
||||
|
||||
def read_skill_file(
|
||||
self,
|
||||
skill_path: str,
|
||||
file_path: str,
|
||||
encoding: str = 'utf-8'
|
||||
) -> str:
|
||||
"""
|
||||
Read a file from a skill directory.
|
||||
|
||||
Args:
|
||||
skill_path: Path to the skill directory (e.g., ./csv-analyzer)
|
||||
file_path: Relative path within the skill (e.g., SKILL.md, scripts/script.py)
|
||||
encoding: File encoding (default: utf-8)
|
||||
|
||||
Returns:
|
||||
File contents or error message
|
||||
"""
|
||||
try:
|
||||
# Resolve skill path
|
||||
skill_path = os.path.expanduser(skill_path)
|
||||
if not os.path.isabs(skill_path):
|
||||
skill_path = os.path.join(self._working_directory, skill_path)
|
||||
skill_path = os.path.abspath(skill_path)
|
||||
|
||||
if not os.path.exists(skill_path):
|
||||
return f"Error: Skill directory not found at {skill_path}"
|
||||
|
||||
if not os.path.isdir(skill_path):
|
||||
return f"Error: {skill_path} is not a directory"
|
||||
|
||||
# Resolve file path within skill
|
||||
full_path = os.path.join(skill_path, file_path)
|
||||
full_path = os.path.abspath(full_path)
|
||||
|
||||
# Security check: ensure file is within skill directory
|
||||
if not full_path.startswith(skill_path):
|
||||
return f"Error: Path traversal detected - {file_path} is outside skill directory"
|
||||
|
||||
if not os.path.exists(full_path):
|
||||
return f"Error: File not found at {full_path}"
|
||||
|
||||
with open(full_path, 'r', encoding=encoding) as f:
|
||||
return f.read()
|
||||
|
||||
except UnicodeDecodeError as e:
|
||||
return f"Error: Could not decode file with {encoding} encoding: {str(e)}"
|
||||
except PermissionError as e:
|
||||
return f"Error: Permission denied reading file: {str(e)}"
|
||||
except Exception as e:
|
||||
return f"Error reading skill file: {str(e)}"
|
||||
|
||||
def list_skill_scripts(self, skill_path: str) -> str:
|
||||
"""
|
||||
List available scripts in a skill's scripts/ directory.
|
||||
|
||||
Args:
|
||||
skill_path: Path to the skill directory
|
||||
|
||||
Returns:
|
||||
JSON string with list of scripts or error message
|
||||
"""
|
||||
import json
|
||||
|
||||
try:
|
||||
# Resolve skill path
|
||||
skill_path = os.path.expanduser(skill_path)
|
||||
if not os.path.isabs(skill_path):
|
||||
skill_path = os.path.join(self._working_directory, skill_path)
|
||||
skill_path = os.path.abspath(skill_path)
|
||||
|
||||
if not os.path.exists(skill_path):
|
||||
return f"Error: Skill directory not found at {skill_path}"
|
||||
|
||||
scripts_dir = os.path.join(skill_path, "scripts")
|
||||
if not os.path.exists(scripts_dir):
|
||||
return json.dumps({"scripts": [], "message": "No scripts/ directory found"})
|
||||
|
||||
scripts = []
|
||||
for item in os.listdir(scripts_dir):
|
||||
item_path = os.path.join(scripts_dir, item)
|
||||
if os.path.isfile(item_path):
|
||||
ext = os.path.splitext(item)[1].lower()
|
||||
if ext in ['.py', '.sh', '.js', '.rb', '.pl']:
|
||||
scripts.append({
|
||||
"name": item,
|
||||
"path": os.path.join("scripts", item),
|
||||
"type": ext[1:] # Remove the dot
|
||||
})
|
||||
|
||||
return json.dumps({"scripts": scripts}, indent=2)
|
||||
|
||||
except Exception as e:
|
||||
return f"Error listing skill scripts: {str(e)}"
|
||||
|
||||
|
||||
# Create default instance for direct function access
|
||||
_skill_tools = SkillTools()
|
||||
|
||||
def run_skill_script(script_path: str, args: str = "", timeout: int = 60) -> str:
|
||||
"""
|
||||
Execute a skill script from a skill's scripts/ directory.
|
||||
|
||||
Args:
|
||||
script_path: Path to the script file (e.g., ./csv-analyzer/scripts/script.py)
|
||||
args: Arguments to pass to the script (e.g., file path to analyze)
|
||||
timeout: Maximum execution time in seconds (default: 60)
|
||||
|
||||
Returns:
|
||||
The output from running the script, or error message if failed
|
||||
"""
|
||||
return _skill_tools.run_skill_script(script_path, args, timeout)
|
||||
|
||||
|
||||
def read_skill_file(skill_path: str, file_path: str, encoding: str = 'utf-8') -> str:
|
||||
"""
|
||||
Read a file from a skill directory.
|
||||
|
||||
Args:
|
||||
skill_path: Path to the skill directory (e.g., ./csv-analyzer)
|
||||
file_path: Relative path within the skill (e.g., SKILL.md, scripts/script.py)
|
||||
encoding: File encoding (default: utf-8)
|
||||
|
||||
Returns:
|
||||
File contents or error message
|
||||
"""
|
||||
return _skill_tools.read_skill_file(skill_path, file_path, encoding)
|
||||
|
||||
|
||||
def list_skill_scripts(skill_path: str) -> str:
|
||||
"""
|
||||
List available scripts in a skill's scripts/ directory.
|
||||
|
||||
Args:
|
||||
skill_path: Path to the skill directory
|
||||
|
||||
Returns:
|
||||
JSON string with list of scripts or error message
|
||||
"""
|
||||
return _skill_tools.list_skill_scripts(skill_path)
|
||||
|
||||
|
||||
def create_skill_tools(working_directory: str = None) -> SkillTools:
|
||||
"""
|
||||
Create a SkillTools instance with a specific working directory.
|
||||
|
||||
This is useful when you need to resolve paths relative to a specific
|
||||
directory rather than the current working directory.
|
||||
|
||||
Args:
|
||||
working_directory: Base directory for resolving relative paths
|
||||
|
||||
Returns:
|
||||
SkillTools instance
|
||||
"""
|
||||
return SkillTools(working_directory)
|
||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "praisonaiagents"
|
||||
version = "0.1.17"
|
||||
version = "0.1.25"
|
||||
description = "Praison AI agents for completing complex tasks with Self Reflection Agents"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Unit tests for PraisonAI Agents evaluation framework."""
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Unit tests for BaseEvaluator class."""
|
||||
|
||||
import pytest
|
||||
import tempfile
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from praisonaiagents.eval.base import BaseEvaluator
|
||||
|
||||
|
||||
class ConcreteEvaluator(BaseEvaluator):
|
||||
"""Concrete implementation for testing."""
|
||||
|
||||
def run(self, **kwargs):
|
||||
return {"test": "result"}
|
||||
|
||||
|
||||
class TestBaseEvaluator:
|
||||
"""Tests for BaseEvaluator abstract class."""
|
||||
|
||||
def test_init_default_values(self):
|
||||
"""Test default initialization values."""
|
||||
evaluator = ConcreteEvaluator()
|
||||
assert evaluator.name.startswith("eval_")
|
||||
assert evaluator.eval_id is not None
|
||||
assert len(evaluator.eval_id) == 8
|
||||
assert evaluator.save_results_path is None
|
||||
assert evaluator.verbose is False
|
||||
|
||||
def test_init_custom_values(self):
|
||||
"""Test initialization with custom values."""
|
||||
evaluator = ConcreteEvaluator(
|
||||
name="my_eval",
|
||||
save_results_path="/tmp/results.json",
|
||||
verbose=True
|
||||
)
|
||||
assert evaluator.name == "my_eval"
|
||||
assert evaluator.save_results_path == "/tmp/results.json"
|
||||
assert evaluator.verbose is True
|
||||
|
||||
def test_eval_id_unique(self):
|
||||
"""Test that each evaluator gets a unique ID."""
|
||||
eval1 = ConcreteEvaluator()
|
||||
eval2 = ConcreteEvaluator()
|
||||
assert eval1.eval_id != eval2.eval_id
|
||||
|
||||
def test_before_run_verbose(self):
|
||||
"""Test before_run hook with verbose logging."""
|
||||
evaluator = ConcreteEvaluator(verbose=True)
|
||||
with patch('praisonaiagents.eval.base.logger') as mock_logger:
|
||||
evaluator.before_run()
|
||||
mock_logger.info.assert_called()
|
||||
|
||||
def test_after_run_verbose(self):
|
||||
"""Test after_run hook with verbose logging."""
|
||||
evaluator = ConcreteEvaluator(verbose=True)
|
||||
with patch('praisonaiagents.eval.base.logger') as mock_logger:
|
||||
evaluator.after_run({"result": "test"})
|
||||
mock_logger.info.assert_called()
|
||||
|
||||
def test_save_result_to_file(self):
|
||||
"""Test saving results to file."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/results.json"
|
||||
evaluator = ConcreteEvaluator(
|
||||
name="test_save",
|
||||
save_results_path=path
|
||||
)
|
||||
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
evaluator._save_result(result)
|
||||
|
||||
assert Path(path).exists()
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
assert data["score"] == 8.0
|
||||
|
||||
def test_save_result_with_placeholders(self):
|
||||
"""Test saving results with path placeholders."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/{{name}}_{{eval_id}}.json"
|
||||
evaluator = ConcreteEvaluator(
|
||||
name="test_eval",
|
||||
save_results_path=path
|
||||
)
|
||||
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
evaluator._save_result(result)
|
||||
|
||||
expected_path = f"{tmpdir}/test_eval_{evaluator.eval_id}.json"
|
||||
assert Path(expected_path).exists()
|
||||
|
||||
def test_save_result_creates_directories(self):
|
||||
"""Test that save creates parent directories."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/nested/dir/results.json"
|
||||
evaluator = ConcreteEvaluator(save_results_path=path)
|
||||
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
evaluator._save_result(result)
|
||||
|
||||
assert Path(path).exists()
|
||||
|
||||
def test_run_abstract_method(self):
|
||||
"""Test that run method works in concrete implementation."""
|
||||
evaluator = ConcreteEvaluator()
|
||||
result = evaluator.run()
|
||||
assert result == {"test": "result"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_async_default(self):
|
||||
"""Test default async run calls sync run."""
|
||||
evaluator = ConcreteEvaluator()
|
||||
result = await evaluator.run_async()
|
||||
assert result == {"test": "result"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_before_run(self):
|
||||
"""Test async before_run hook."""
|
||||
evaluator = ConcreteEvaluator(verbose=True)
|
||||
with patch('praisonaiagents.eval.base.logger') as mock_logger:
|
||||
await evaluator.async_before_run()
|
||||
mock_logger.info.assert_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_after_run(self):
|
||||
"""Test async after_run hook."""
|
||||
evaluator = ConcreteEvaluator(verbose=True)
|
||||
with patch('praisonaiagents.eval.base.logger') as mock_logger:
|
||||
await evaluator.async_after_run({"result": "test"})
|
||||
mock_logger.info.assert_called()
|
||||
|
||||
def test_repr(self):
|
||||
"""Test string representation."""
|
||||
evaluator = ConcreteEvaluator(name="test_eval")
|
||||
repr_str = repr(evaluator)
|
||||
assert "ConcreteEvaluator" in repr_str
|
||||
assert "test_eval" in repr_str
|
||||
assert evaluator.eval_id in repr_str
|
||||
@@ -0,0 +1,425 @@
|
||||
"""Unit tests for evaluator classes."""
|
||||
|
||||
import pytest
|
||||
import time
|
||||
from unittest.mock import MagicMock, patch, AsyncMock
|
||||
|
||||
from praisonaiagents.eval.accuracy import AccuracyEvaluator
|
||||
from praisonaiagents.eval.performance import PerformanceEvaluator
|
||||
from praisonaiagents.eval.reliability import ReliabilityEvaluator
|
||||
from praisonaiagents.eval.criteria import CriteriaEvaluator
|
||||
from praisonaiagents.eval.results import (
|
||||
AccuracyResult,
|
||||
PerformanceResult,
|
||||
ReliabilityResult,
|
||||
CriteriaResult,
|
||||
)
|
||||
|
||||
|
||||
class TestAccuracyEvaluator:
|
||||
"""Tests for AccuracyEvaluator class."""
|
||||
|
||||
def test_init_with_func(self):
|
||||
"""Test initialization with function."""
|
||||
func = lambda x: "test output"
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=func,
|
||||
input_text="test input",
|
||||
expected_output="test output"
|
||||
)
|
||||
assert evaluator.func == func
|
||||
assert evaluator.input_text == "test input"
|
||||
assert evaluator.expected_output == "test output"
|
||||
|
||||
def test_init_with_agent(self):
|
||||
"""Test initialization with agent."""
|
||||
agent = MagicMock()
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
assert evaluator.agent == agent
|
||||
|
||||
def test_init_requires_agent_or_func(self):
|
||||
"""Test that either agent or func must be provided."""
|
||||
with pytest.raises(ValueError, match="Either 'agent' or 'func' must be provided"):
|
||||
AccuracyEvaluator(
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
|
||||
def test_get_output_from_func(self):
|
||||
"""Test getting output from function."""
|
||||
func = lambda x: f"output for {x}"
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=func,
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
output = evaluator._get_output()
|
||||
assert output == "output for test"
|
||||
|
||||
def test_get_output_from_agent_chat(self):
|
||||
"""Test getting output from agent with chat method."""
|
||||
agent = MagicMock()
|
||||
agent.chat.return_value = "agent response"
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
output = evaluator._get_output()
|
||||
assert output == "agent response"
|
||||
agent.chat.assert_called_once_with("test")
|
||||
|
||||
def test_get_output_from_agent_start(self):
|
||||
"""Test getting output from agent with start method."""
|
||||
agent = MagicMock(spec=['start'])
|
||||
result = MagicMock()
|
||||
result.raw = "agent result"
|
||||
agent.start.return_value = result
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_output="expected"
|
||||
)
|
||||
output = evaluator._get_output()
|
||||
assert output == "agent result"
|
||||
|
||||
def test_judge_output(self):
|
||||
"""Test LLM judging of output."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 8\nREASONING: Good match"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "4"
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=test_func,
|
||||
input_text="What is 2+2?",
|
||||
expected_output="4"
|
||||
)
|
||||
|
||||
score = evaluator._judge_output("4")
|
||||
assert score.score == 8.0
|
||||
assert "Good match" in score.reasoning
|
||||
|
||||
def test_run_returns_accuracy_result(self):
|
||||
"""Test that run returns AccuracyResult."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 9\nREASONING: Excellent"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "4"
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=test_func,
|
||||
input_text="What is 2+2?",
|
||||
expected_output="4",
|
||||
num_iterations=2
|
||||
)
|
||||
|
||||
result = evaluator.run()
|
||||
assert isinstance(result, AccuracyResult)
|
||||
assert len(result.evaluations) == 2
|
||||
|
||||
def test_evaluate_output(self):
|
||||
"""Test evaluate_output method for pre-generated outputs."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 10\nREASONING: Perfect"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "unused"
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
func=test_func,
|
||||
input_text="What is 2+2?",
|
||||
expected_output="4"
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_output("4")
|
||||
assert isinstance(result, AccuracyResult)
|
||||
assert len(result.evaluations) == 1
|
||||
assert result.evaluations[0].score == 10.0
|
||||
|
||||
|
||||
class TestPerformanceEvaluator:
|
||||
"""Tests for PerformanceEvaluator class."""
|
||||
|
||||
def test_init_with_func(self):
|
||||
"""Test initialization with function."""
|
||||
func = lambda: time.sleep(0.01)
|
||||
evaluator = PerformanceEvaluator(
|
||||
func=func,
|
||||
num_iterations=5,
|
||||
warmup_runs=2
|
||||
)
|
||||
assert evaluator.func == func
|
||||
assert evaluator.num_iterations == 5
|
||||
assert evaluator.warmup_runs == 2
|
||||
|
||||
def test_init_with_agent(self):
|
||||
"""Test initialization with agent."""
|
||||
agent = MagicMock()
|
||||
evaluator = PerformanceEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
num_iterations=3
|
||||
)
|
||||
assert evaluator.agent == agent
|
||||
assert evaluator.input_text == "test"
|
||||
|
||||
def test_init_requires_func_or_agent(self):
|
||||
"""Test that either func or agent must be provided."""
|
||||
with pytest.raises(ValueError, match="Either 'func' or 'agent' must be provided"):
|
||||
PerformanceEvaluator()
|
||||
|
||||
def test_run_measures_time(self):
|
||||
"""Test that run measures execution time."""
|
||||
call_count = 0
|
||||
def slow_func():
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
time.sleep(0.01)
|
||||
|
||||
evaluator = PerformanceEvaluator(
|
||||
func=slow_func,
|
||||
num_iterations=3,
|
||||
warmup_runs=1,
|
||||
track_memory=False
|
||||
)
|
||||
|
||||
result = evaluator.run()
|
||||
assert isinstance(result, PerformanceResult)
|
||||
assert len(result.metrics) == 3
|
||||
assert all(m.run_time >= 0.01 for m in result.metrics)
|
||||
assert call_count == 4 # 1 warmup + 3 iterations
|
||||
|
||||
def test_run_with_agent(self):
|
||||
"""Test run with agent."""
|
||||
agent = MagicMock()
|
||||
agent.chat.return_value = "response"
|
||||
|
||||
evaluator = PerformanceEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
num_iterations=2,
|
||||
warmup_runs=1,
|
||||
track_memory=False
|
||||
)
|
||||
|
||||
result = evaluator.run()
|
||||
assert isinstance(result, PerformanceResult)
|
||||
assert len(result.metrics) == 2
|
||||
assert agent.chat.call_count == 3 # 1 warmup + 2 iterations
|
||||
|
||||
|
||||
class TestReliabilityEvaluator:
|
||||
"""Tests for ReliabilityEvaluator class."""
|
||||
|
||||
def test_init(self):
|
||||
"""Test initialization."""
|
||||
agent = MagicMock()
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_tools=["search", "calculate"]
|
||||
)
|
||||
assert evaluator.agent == agent
|
||||
assert evaluator.expected_tools == {"search", "calculate"}
|
||||
|
||||
def test_init_requires_agent(self):
|
||||
"""Test that agent must be provided."""
|
||||
with pytest.raises(ValueError, match="'agent' must be provided"):
|
||||
ReliabilityEvaluator(
|
||||
input_text="test",
|
||||
expected_tools=["search"]
|
||||
)
|
||||
|
||||
def test_evaluate_tool_calls(self):
|
||||
"""Test evaluate_tool_calls method."""
|
||||
agent = MagicMock()
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_tools=["search", "calculate"],
|
||||
forbidden_tools=["delete"]
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_tool_calls(["search", "calculate"])
|
||||
assert isinstance(result, ReliabilityResult)
|
||||
assert result.passed is True
|
||||
assert len(result.passed_calls) == 3 # 2 expected + 1 forbidden not called
|
||||
|
||||
def test_evaluate_tool_calls_with_failures(self):
|
||||
"""Test evaluate_tool_calls with missing tools."""
|
||||
agent = MagicMock()
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_tools=["search", "calculate"]
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_tool_calls(["search"]) # missing calculate
|
||||
assert result.passed is False
|
||||
assert len(result.failed_calls) == 1
|
||||
|
||||
def test_forbidden_tools(self):
|
||||
"""Test forbidden tools detection."""
|
||||
agent = MagicMock()
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text="test",
|
||||
expected_tools=[],
|
||||
forbidden_tools=["delete", "drop"]
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_tool_calls(["delete"]) # called forbidden tool
|
||||
assert result.passed is False
|
||||
assert len(result.failed_calls) == 1
|
||||
|
||||
|
||||
class TestCriteriaEvaluator:
|
||||
"""Tests for CriteriaEvaluator class."""
|
||||
|
||||
def test_init(self):
|
||||
"""Test initialization."""
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Response is helpful",
|
||||
func=lambda x: "test",
|
||||
input_text="test"
|
||||
)
|
||||
assert evaluator.criteria == "Response is helpful"
|
||||
assert evaluator.scoring_type == "numeric"
|
||||
assert evaluator.threshold == 7.0
|
||||
|
||||
def test_init_binary_scoring(self):
|
||||
"""Test initialization with binary scoring."""
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Response is helpful",
|
||||
func=lambda x: "test",
|
||||
scoring_type="binary"
|
||||
)
|
||||
assert evaluator.scoring_type == "binary"
|
||||
|
||||
def test_judge_output_numeric(self):
|
||||
"""Test numeric scoring."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 8\nREASONING: Good"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "test"
|
||||
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Be helpful",
|
||||
func=test_func,
|
||||
scoring_type="numeric",
|
||||
threshold=7.0
|
||||
)
|
||||
|
||||
score = evaluator._judge_output("test output")
|
||||
assert score.score == 8.0
|
||||
assert score.passed is True
|
||||
|
||||
def test_judge_output_binary(self):
|
||||
"""Test binary scoring."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "RESULT: PASS\nREASONING: Meets criteria"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "test"
|
||||
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Be helpful",
|
||||
func=test_func,
|
||||
scoring_type="binary"
|
||||
)
|
||||
|
||||
score = evaluator._judge_output("test output")
|
||||
assert score.score == 10.0
|
||||
assert score.passed is True
|
||||
|
||||
def test_on_fail_callback(self):
|
||||
"""Test on_fail callback is called."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 3\nREASONING: Poor"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
on_fail_called = []
|
||||
def on_fail(score):
|
||||
on_fail_called.append(score)
|
||||
|
||||
def test_func(x):
|
||||
return "test"
|
||||
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Be helpful",
|
||||
func=test_func,
|
||||
threshold=7.0,
|
||||
on_fail=on_fail
|
||||
)
|
||||
|
||||
evaluator.run()
|
||||
assert len(on_fail_called) == 1
|
||||
assert on_fail_called[0].score == 3.0
|
||||
|
||||
def test_evaluate_output_criteria(self):
|
||||
"""Test evaluate_output method."""
|
||||
with patch('openai.OpenAI') as mock_openai_class:
|
||||
mock_client = MagicMock()
|
||||
mock_openai_class.return_value = mock_client
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "SCORE: 9\nREASONING: Great"
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
def test_func(x):
|
||||
return "unused"
|
||||
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria="Be helpful",
|
||||
func=test_func
|
||||
)
|
||||
|
||||
result = evaluator.evaluate_output("pre-generated output")
|
||||
assert isinstance(result, CriteriaResult)
|
||||
assert len(result.evaluations) == 1
|
||||
@@ -0,0 +1,359 @@
|
||||
"""Unit tests for evaluation result dataclasses."""
|
||||
|
||||
import pytest
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
from praisonaiagents.eval.results import (
|
||||
EvaluationScore,
|
||||
AccuracyResult,
|
||||
PerformanceMetrics,
|
||||
PerformanceResult,
|
||||
ToolCallResult,
|
||||
ReliabilityResult,
|
||||
CriteriaScore,
|
||||
CriteriaResult,
|
||||
)
|
||||
|
||||
|
||||
class TestEvaluationScore:
|
||||
"""Tests for EvaluationScore dataclass."""
|
||||
|
||||
def test_creation(self):
|
||||
"""Test basic creation of EvaluationScore."""
|
||||
score = EvaluationScore(
|
||||
score=8.5,
|
||||
reasoning="Good response",
|
||||
input_text="What is 2+2?",
|
||||
output_text="4",
|
||||
expected_output="4"
|
||||
)
|
||||
assert score.score == 8.5
|
||||
assert score.reasoning == "Good response"
|
||||
assert score.input_text == "What is 2+2?"
|
||||
assert score.output_text == "4"
|
||||
assert score.expected_output == "4"
|
||||
|
||||
def test_timestamp_auto_generated(self):
|
||||
"""Test that timestamp is auto-generated."""
|
||||
score = EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Test",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
)
|
||||
assert score.timestamp is not None
|
||||
datetime.fromisoformat(score.timestamp)
|
||||
|
||||
def test_metadata_default(self):
|
||||
"""Test that metadata defaults to empty dict."""
|
||||
score = EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Test",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
)
|
||||
assert score.metadata == {}
|
||||
|
||||
|
||||
class TestAccuracyResult:
|
||||
"""Tests for AccuracyResult dataclass."""
|
||||
|
||||
def test_empty_result(self):
|
||||
"""Test empty AccuracyResult."""
|
||||
result = AccuracyResult()
|
||||
assert result.evaluations == []
|
||||
assert result.avg_score == 0.0
|
||||
assert result.min_score == 0.0
|
||||
assert result.max_score == 0.0
|
||||
assert result.std_dev == 0.0
|
||||
assert result.passed is False
|
||||
|
||||
def test_single_evaluation(self):
|
||||
"""Test AccuracyResult with single evaluation."""
|
||||
result = AccuracyResult()
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Good",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
assert result.avg_score == 8.0
|
||||
assert result.min_score == 8.0
|
||||
assert result.max_score == 8.0
|
||||
assert result.std_dev == 0.0
|
||||
assert result.passed is True
|
||||
|
||||
def test_multiple_evaluations(self):
|
||||
"""Test AccuracyResult with multiple evaluations."""
|
||||
result = AccuracyResult()
|
||||
for score in [6.0, 8.0, 10.0]:
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=score,
|
||||
reasoning="Test",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
assert result.avg_score == 8.0
|
||||
assert result.min_score == 6.0
|
||||
assert result.max_score == 10.0
|
||||
assert result.passed is True
|
||||
|
||||
def test_passed_threshold(self):
|
||||
"""Test passed property with different scores."""
|
||||
result = AccuracyResult()
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=6.0,
|
||||
reasoning="Below threshold",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
assert result.passed is False
|
||||
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Above threshold",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
assert result.passed is True
|
||||
|
||||
def test_to_dict(self):
|
||||
"""Test to_dict method."""
|
||||
result = AccuracyResult(eval_id="test123", name="test_eval")
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Good",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
d = result.to_dict()
|
||||
assert d["eval_id"] == "test123"
|
||||
assert d["name"] == "test_eval"
|
||||
assert d["avg_score"] == 8.0
|
||||
assert d["num_evaluations"] == 1
|
||||
|
||||
def test_to_json(self):
|
||||
"""Test to_json method."""
|
||||
result = AccuracyResult(eval_id="test123")
|
||||
result.evaluations.append(EvaluationScore(
|
||||
score=8.0,
|
||||
reasoning="Good",
|
||||
input_text="test",
|
||||
output_text="test"
|
||||
))
|
||||
json_str = result.to_json()
|
||||
data = json.loads(json_str)
|
||||
assert data["eval_id"] == "test123"
|
||||
|
||||
|
||||
class TestPerformanceMetrics:
|
||||
"""Tests for PerformanceMetrics dataclass."""
|
||||
|
||||
def test_creation(self):
|
||||
"""Test basic creation."""
|
||||
metrics = PerformanceMetrics(
|
||||
run_time=0.5,
|
||||
memory_usage=10.5,
|
||||
iteration=1
|
||||
)
|
||||
assert metrics.run_time == 0.5
|
||||
assert metrics.memory_usage == 10.5
|
||||
assert metrics.iteration == 1
|
||||
|
||||
|
||||
class TestPerformanceResult:
|
||||
"""Tests for PerformanceResult dataclass."""
|
||||
|
||||
def test_empty_result(self):
|
||||
"""Test empty PerformanceResult."""
|
||||
result = PerformanceResult()
|
||||
assert result.metrics == []
|
||||
assert result.avg_run_time == 0.0
|
||||
assert result.min_run_time == 0.0
|
||||
assert result.max_run_time == 0.0
|
||||
assert result.avg_memory == 0.0
|
||||
|
||||
def test_with_metrics(self):
|
||||
"""Test PerformanceResult with metrics."""
|
||||
result = PerformanceResult()
|
||||
for i, (time, mem) in enumerate([(0.1, 5.0), (0.2, 10.0), (0.3, 15.0)]):
|
||||
result.metrics.append(PerformanceMetrics(
|
||||
run_time=time,
|
||||
memory_usage=mem,
|
||||
iteration=i + 1
|
||||
))
|
||||
assert result.avg_run_time == 0.2
|
||||
assert result.min_run_time == 0.1
|
||||
assert result.max_run_time == 0.3
|
||||
assert result.avg_memory == 10.0
|
||||
assert result.max_memory == 15.0
|
||||
|
||||
def test_median_and_p95(self):
|
||||
"""Test median and p95 calculations."""
|
||||
result = PerformanceResult()
|
||||
for i in range(100):
|
||||
result.metrics.append(PerformanceMetrics(
|
||||
run_time=float(i + 1) / 100,
|
||||
memory_usage=1.0,
|
||||
iteration=i + 1
|
||||
))
|
||||
assert 0.49 <= result.median_run_time <= 0.51
|
||||
assert result.p95_run_time >= 0.95
|
||||
|
||||
def test_to_dict(self):
|
||||
"""Test to_dict method."""
|
||||
result = PerformanceResult(warmup_runs=5, eval_id="perf123")
|
||||
result.metrics.append(PerformanceMetrics(
|
||||
run_time=0.5,
|
||||
memory_usage=10.0,
|
||||
iteration=1
|
||||
))
|
||||
d = result.to_dict()
|
||||
assert d["warmup_runs"] == 5
|
||||
assert d["eval_id"] == "perf123"
|
||||
assert d["num_iterations"] == 1
|
||||
|
||||
|
||||
class TestToolCallResult:
|
||||
"""Tests for ToolCallResult dataclass."""
|
||||
|
||||
def test_creation(self):
|
||||
"""Test basic creation."""
|
||||
tc = ToolCallResult(
|
||||
tool_name="search",
|
||||
expected=True,
|
||||
actual=True,
|
||||
passed=True
|
||||
)
|
||||
assert tc.tool_name == "search"
|
||||
assert tc.expected is True
|
||||
assert tc.actual is True
|
||||
assert tc.passed is True
|
||||
|
||||
|
||||
class TestReliabilityResult:
|
||||
"""Tests for ReliabilityResult dataclass."""
|
||||
|
||||
def test_empty_result(self):
|
||||
"""Test empty ReliabilityResult."""
|
||||
result = ReliabilityResult()
|
||||
assert result.tool_results == []
|
||||
assert result.pass_rate == 0.0
|
||||
assert result.status == "PASSED"
|
||||
assert result.passed is True
|
||||
|
||||
def test_with_tool_calls(self):
|
||||
"""Test ReliabilityResult with tool calls."""
|
||||
result = ReliabilityResult()
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name="search",
|
||||
expected=True,
|
||||
actual=True,
|
||||
passed=True
|
||||
))
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name="calculate",
|
||||
expected=True,
|
||||
actual=False,
|
||||
passed=False
|
||||
))
|
||||
assert len(result.passed_calls) == 1
|
||||
assert len(result.failed_calls) == 1
|
||||
assert result.pass_rate == 0.5
|
||||
assert result.status == "FAILED"
|
||||
assert result.passed is False
|
||||
|
||||
def test_all_passed(self):
|
||||
"""Test when all tool calls pass."""
|
||||
result = ReliabilityResult()
|
||||
for name in ["tool1", "tool2", "tool3"]:
|
||||
result.tool_results.append(ToolCallResult(
|
||||
tool_name=name,
|
||||
expected=True,
|
||||
actual=True,
|
||||
passed=True
|
||||
))
|
||||
assert result.pass_rate == 1.0
|
||||
assert result.passed is True
|
||||
|
||||
|
||||
class TestCriteriaScore:
|
||||
"""Tests for CriteriaScore dataclass."""
|
||||
|
||||
def test_creation(self):
|
||||
"""Test basic creation."""
|
||||
score = CriteriaScore(
|
||||
score=8.0,
|
||||
passed=True,
|
||||
reasoning="Meets criteria",
|
||||
output_text="test output",
|
||||
criteria="Be helpful"
|
||||
)
|
||||
assert score.score == 8.0
|
||||
assert score.passed is True
|
||||
assert score.reasoning == "Meets criteria"
|
||||
|
||||
|
||||
class TestCriteriaResult:
|
||||
"""Tests for CriteriaResult dataclass."""
|
||||
|
||||
def test_empty_result(self):
|
||||
"""Test empty CriteriaResult."""
|
||||
result = CriteriaResult(criteria="Be helpful")
|
||||
assert result.evaluations == []
|
||||
assert result.avg_score == 0.0
|
||||
assert result.pass_rate == 0.0
|
||||
|
||||
def test_numeric_scoring(self):
|
||||
"""Test numeric scoring mode."""
|
||||
result = CriteriaResult(
|
||||
criteria="Be helpful",
|
||||
scoring_type="numeric",
|
||||
threshold=7.0
|
||||
)
|
||||
result.evaluations.append(CriteriaScore(
|
||||
score=8.0,
|
||||
passed=True,
|
||||
reasoning="Good",
|
||||
output_text="test",
|
||||
criteria="Be helpful"
|
||||
))
|
||||
assert result.avg_score == 8.0
|
||||
assert result.passed is True
|
||||
|
||||
def test_binary_scoring(self):
|
||||
"""Test binary scoring mode."""
|
||||
result = CriteriaResult(
|
||||
criteria="Be helpful",
|
||||
scoring_type="binary"
|
||||
)
|
||||
result.evaluations.append(CriteriaScore(
|
||||
score=10.0,
|
||||
passed=True,
|
||||
reasoning="Pass",
|
||||
output_text="test",
|
||||
criteria="Be helpful"
|
||||
))
|
||||
result.evaluations.append(CriteriaScore(
|
||||
score=0.0,
|
||||
passed=False,
|
||||
reasoning="Fail",
|
||||
output_text="test",
|
||||
criteria="Be helpful"
|
||||
))
|
||||
assert result.pass_rate == 0.5
|
||||
assert result.passed is True
|
||||
|
||||
def test_to_dict(self):
|
||||
"""Test to_dict method."""
|
||||
result = CriteriaResult(
|
||||
criteria="Be helpful",
|
||||
scoring_type="numeric",
|
||||
threshold=7.0,
|
||||
eval_id="crit123"
|
||||
)
|
||||
d = result.to_dict()
|
||||
assert d["criteria"] == "Be helpful"
|
||||
assert d["scoring_type"] == "numeric"
|
||||
assert d["threshold"] == 7.0
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Unit tests for evaluation utility functions."""
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from praisonaiagents.eval.utils import (
|
||||
save_result_to_file,
|
||||
load_result_from_file,
|
||||
format_score,
|
||||
format_percentage,
|
||||
format_duration,
|
||||
format_memory,
|
||||
calculate_pass_rate,
|
||||
)
|
||||
|
||||
|
||||
class TestSaveResultToFile:
|
||||
"""Tests for save_result_to_file function."""
|
||||
|
||||
def test_save_with_to_dict(self):
|
||||
"""Test saving result with to_dict method."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/result.json"
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0, "passed": True}
|
||||
|
||||
success = save_result_to_file(result, path)
|
||||
|
||||
assert success is True
|
||||
assert Path(path).exists()
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
assert data["score"] == 8.0
|
||||
|
||||
def test_save_with_dict_attr(self):
|
||||
"""Test saving result with __dict__ attribute."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/result.json"
|
||||
|
||||
class SimpleResult:
|
||||
def __init__(self):
|
||||
self.score = 9.0
|
||||
self.name = "test"
|
||||
|
||||
result = SimpleResult()
|
||||
success = save_result_to_file(result, path)
|
||||
|
||||
assert success is True
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
assert data["score"] == 9.0
|
||||
|
||||
def test_save_with_placeholders(self):
|
||||
"""Test saving with path placeholders."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/{{name}}_{{eval_id}}.json"
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
success = save_result_to_file(
|
||||
result, path,
|
||||
eval_id="abc123",
|
||||
name="test_eval"
|
||||
)
|
||||
|
||||
assert success is True
|
||||
expected_path = f"{tmpdir}/test_eval_abc123.json"
|
||||
assert Path(expected_path).exists()
|
||||
|
||||
def test_save_creates_directories(self):
|
||||
"""Test that save creates parent directories."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/nested/dir/result.json"
|
||||
result = MagicMock()
|
||||
result.to_dict.return_value = {"score": 8.0}
|
||||
|
||||
success = save_result_to_file(result, path)
|
||||
|
||||
assert success is True
|
||||
assert Path(path).exists()
|
||||
|
||||
def test_save_failure_returns_false(self):
|
||||
"""Test that save failure returns False."""
|
||||
result = MagicMock()
|
||||
result.to_dict.side_effect = Exception("Test error")
|
||||
|
||||
success = save_result_to_file(result, "/invalid/path/result.json")
|
||||
|
||||
assert success is False
|
||||
|
||||
|
||||
class TestLoadResultFromFile:
|
||||
"""Tests for load_result_from_file function."""
|
||||
|
||||
def test_load_existing_file(self):
|
||||
"""Test loading existing result file."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/result.json"
|
||||
data = {"score": 8.0, "passed": True}
|
||||
with open(path, 'w') as f:
|
||||
json.dump(data, f)
|
||||
|
||||
loaded = load_result_from_file(path)
|
||||
|
||||
assert loaded == data
|
||||
|
||||
def test_load_nonexistent_file(self):
|
||||
"""Test loading nonexistent file returns None."""
|
||||
result = load_result_from_file("/nonexistent/path/result.json")
|
||||
assert result is None
|
||||
|
||||
def test_load_invalid_json(self):
|
||||
"""Test loading invalid JSON returns None."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = f"{tmpdir}/invalid.json"
|
||||
with open(path, 'w') as f:
|
||||
f.write("not valid json")
|
||||
|
||||
result = load_result_from_file(path)
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestFormatScore:
|
||||
"""Tests for format_score function."""
|
||||
|
||||
def test_format_default_max(self):
|
||||
"""Test formatting with default max score."""
|
||||
assert format_score(8.5) == "8.5/10"
|
||||
|
||||
def test_format_custom_max(self):
|
||||
"""Test formatting with custom max score."""
|
||||
assert format_score(85.0, 100.0) == "85.0/100"
|
||||
|
||||
def test_format_zero(self):
|
||||
"""Test formatting zero score."""
|
||||
assert format_score(0.0) == "0.0/10"
|
||||
|
||||
|
||||
class TestFormatPercentage:
|
||||
"""Tests for format_percentage function."""
|
||||
|
||||
def test_format_full(self):
|
||||
"""Test formatting 100%."""
|
||||
assert format_percentage(1.0) == "100.0%"
|
||||
|
||||
def test_format_half(self):
|
||||
"""Test formatting 50%."""
|
||||
assert format_percentage(0.5) == "50.0%"
|
||||
|
||||
def test_format_zero(self):
|
||||
"""Test formatting 0%."""
|
||||
assert format_percentage(0.0) == "0.0%"
|
||||
|
||||
def test_format_decimal(self):
|
||||
"""Test formatting decimal percentage."""
|
||||
assert format_percentage(0.333) == "33.3%"
|
||||
|
||||
|
||||
class TestFormatDuration:
|
||||
"""Tests for format_duration function."""
|
||||
|
||||
def test_format_microseconds(self):
|
||||
"""Test formatting microseconds."""
|
||||
result = format_duration(0.0001)
|
||||
assert "μs" in result
|
||||
|
||||
def test_format_milliseconds(self):
|
||||
"""Test formatting milliseconds."""
|
||||
result = format_duration(0.5)
|
||||
assert "ms" in result
|
||||
|
||||
def test_format_seconds(self):
|
||||
"""Test formatting seconds."""
|
||||
result = format_duration(5.5)
|
||||
assert "s" in result
|
||||
assert "m" not in result
|
||||
|
||||
def test_format_minutes(self):
|
||||
"""Test formatting minutes."""
|
||||
result = format_duration(90.0)
|
||||
assert "m" in result
|
||||
|
||||
|
||||
class TestFormatMemory:
|
||||
"""Tests for format_memory function."""
|
||||
|
||||
def test_format_kilobytes(self):
|
||||
"""Test formatting kilobytes."""
|
||||
result = format_memory(0.5)
|
||||
assert "KB" in result
|
||||
|
||||
def test_format_megabytes(self):
|
||||
"""Test formatting megabytes."""
|
||||
result = format_memory(100.0)
|
||||
assert "MB" in result
|
||||
|
||||
def test_format_gigabytes(self):
|
||||
"""Test formatting gigabytes."""
|
||||
result = format_memory(2048.0)
|
||||
assert "GB" in result
|
||||
|
||||
|
||||
class TestCalculatePassRate:
|
||||
"""Tests for calculate_pass_rate function."""
|
||||
|
||||
def test_all_passed(self):
|
||||
"""Test 100% pass rate."""
|
||||
assert calculate_pass_rate(10, 10) == 1.0
|
||||
|
||||
def test_none_passed(self):
|
||||
"""Test 0% pass rate."""
|
||||
assert calculate_pass_rate(0, 10) == 0.0
|
||||
|
||||
def test_half_passed(self):
|
||||
"""Test 50% pass rate."""
|
||||
assert calculate_pass_rate(5, 10) == 0.5
|
||||
|
||||
def test_zero_total(self):
|
||||
"""Test with zero total."""
|
||||
assert calculate_pass_rate(0, 0) == 0.0
|
||||
@@ -0,0 +1 @@
|
||||
"""Unit tests for Agent Skills module."""
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Tests for Agent Skills discovery."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestGetDefaultSkillDirs:
|
||||
"""Tests for get_default_skill_dirs function."""
|
||||
|
||||
def test_returns_list(self):
|
||||
"""Test that function returns a list."""
|
||||
from praisonaiagents.skills.discovery import get_default_skill_dirs
|
||||
|
||||
result = get_default_skill_dirs()
|
||||
|
||||
assert isinstance(result, list)
|
||||
|
||||
def test_returns_path_objects(self):
|
||||
"""Test that returned items are Path objects."""
|
||||
from praisonaiagents.skills.discovery import get_default_skill_dirs
|
||||
|
||||
result = get_default_skill_dirs()
|
||||
|
||||
for item in result:
|
||||
assert isinstance(item, Path)
|
||||
|
||||
|
||||
class TestDiscoverSkills:
|
||||
"""Tests for discover_skills function."""
|
||||
|
||||
def test_discover_empty_directory(self):
|
||||
"""Test discovering skills in empty directory."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
result = discover_skills([tmpdir], include_defaults=False)
|
||||
|
||||
assert result == []
|
||||
|
||||
def test_discover_single_skill(self):
|
||||
"""Test discovering a single valid skill."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create a skill directory
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill for discovery
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
""")
|
||||
|
||||
result = discover_skills([tmpdir], include_defaults=False)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].name == "test-skill"
|
||||
assert result[0].description == "A test skill for discovery"
|
||||
|
||||
def test_discover_multiple_skills(self):
|
||||
"""Test discovering multiple skills."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create first skill
|
||||
skill1_dir = Path(tmpdir) / "skill-one"
|
||||
skill1_dir.mkdir()
|
||||
(skill1_dir / "SKILL.md").write_text("""---
|
||||
name: skill-one
|
||||
description: First skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create second skill
|
||||
skill2_dir = Path(tmpdir) / "skill-two"
|
||||
skill2_dir.mkdir()
|
||||
(skill2_dir / "SKILL.md").write_text("""---
|
||||
name: skill-two
|
||||
description: Second skill
|
||||
---
|
||||
""")
|
||||
|
||||
result = discover_skills([tmpdir], include_defaults=False)
|
||||
|
||||
assert len(result) == 2
|
||||
names = {s.name for s in result}
|
||||
assert "skill-one" in names
|
||||
assert "skill-two" in names
|
||||
|
||||
def test_discover_skips_invalid_skills(self):
|
||||
"""Test that invalid skills are skipped."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create valid skill
|
||||
valid_dir = Path(tmpdir) / "valid-skill"
|
||||
valid_dir.mkdir()
|
||||
(valid_dir / "SKILL.md").write_text("""---
|
||||
name: valid-skill
|
||||
description: A valid skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create invalid skill (no SKILL.md)
|
||||
invalid_dir = Path(tmpdir) / "invalid-skill"
|
||||
invalid_dir.mkdir()
|
||||
|
||||
result = discover_skills([tmpdir], include_defaults=False)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].name == "valid-skill"
|
||||
|
||||
def test_discover_from_multiple_directories(self):
|
||||
"""Test discovering from multiple directories."""
|
||||
from praisonaiagents.skills.discovery import discover_skills
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir1:
|
||||
with tempfile.TemporaryDirectory() as tmpdir2:
|
||||
# Create skill in first directory
|
||||
skill1_dir = Path(tmpdir1) / "skill-a"
|
||||
skill1_dir.mkdir()
|
||||
(skill1_dir / "SKILL.md").write_text("""---
|
||||
name: skill-a
|
||||
description: Skill A
|
||||
---
|
||||
""")
|
||||
|
||||
# Create skill in second directory
|
||||
skill2_dir = Path(tmpdir2) / "skill-b"
|
||||
skill2_dir.mkdir()
|
||||
(skill2_dir / "SKILL.md").write_text("""---
|
||||
name: skill-b
|
||||
description: Skill B
|
||||
---
|
||||
""")
|
||||
|
||||
result = discover_skills([tmpdir1, tmpdir2], include_defaults=False)
|
||||
|
||||
assert len(result) == 2
|
||||
|
||||
|
||||
class TestDiscoverSkill:
|
||||
"""Tests for discover_skill function (single skill)."""
|
||||
|
||||
def test_discover_valid_skill(self):
|
||||
"""Test discovering a single valid skill."""
|
||||
from praisonaiagents.skills.discovery import discover_skill
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "my-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: my-skill
|
||||
description: My skill
|
||||
---
|
||||
""")
|
||||
|
||||
result = discover_skill(str(skill_dir))
|
||||
|
||||
assert result is not None
|
||||
assert result.name == "my-skill"
|
||||
|
||||
def test_discover_nonexistent_path(self):
|
||||
"""Test discovering from nonexistent path."""
|
||||
from praisonaiagents.skills.discovery import discover_skill
|
||||
|
||||
result = discover_skill("/nonexistent/path")
|
||||
|
||||
assert result is None
|
||||
|
||||
def test_discover_directory_without_skill_md(self):
|
||||
"""Test discovering from directory without SKILL.md."""
|
||||
from praisonaiagents.skills.discovery import discover_skill
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
result = discover_skill(tmpdir)
|
||||
|
||||
assert result is None
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Tests for Agent Skills loader."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestSkillLoader:
|
||||
"""Tests for SkillLoader class."""
|
||||
|
||||
def test_load_metadata_valid_skill(self):
|
||||
"""Test loading metadata from valid skill."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
# Instructions
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
|
||||
assert skill is not None
|
||||
assert skill.properties.name == "test-skill"
|
||||
assert skill.instructions is None # Not activated yet
|
||||
|
||||
def test_load_metadata_nonexistent(self):
|
||||
"""Test loading from nonexistent path."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata("/nonexistent/path")
|
||||
|
||||
assert skill is None
|
||||
|
||||
def test_activate_skill(self):
|
||||
"""Test activating a skill loads instructions."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
# Instructions
|
||||
|
||||
Follow these steps:
|
||||
1. Do this
|
||||
2. Do that
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
|
||||
assert not skill.is_activated
|
||||
|
||||
result = loader.activate(skill)
|
||||
|
||||
assert result is True
|
||||
assert skill.is_activated
|
||||
assert "Follow these steps" in skill.instructions
|
||||
|
||||
def test_activate_already_activated(self):
|
||||
"""Test activating already activated skill returns True."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
Instructions.
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
loader.activate(skill)
|
||||
|
||||
# Activate again
|
||||
result = loader.activate(skill)
|
||||
|
||||
assert result is True
|
||||
|
||||
def test_load_scripts(self):
|
||||
"""Test loading scripts from skill."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create scripts directory
|
||||
scripts_dir = skill_dir / "scripts"
|
||||
scripts_dir.mkdir()
|
||||
(scripts_dir / "helper.py").write_text("print('hello')")
|
||||
(scripts_dir / "utils.sh").write_text("echo 'hello'")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
scripts = loader.load_scripts(skill)
|
||||
|
||||
assert "helper.py" in scripts
|
||||
assert "utils.sh" in scripts
|
||||
assert "print('hello')" in scripts["helper.py"]
|
||||
|
||||
def test_load_references(self):
|
||||
"""Test loading references from skill."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create references directory
|
||||
refs_dir = skill_dir / "references"
|
||||
refs_dir.mkdir()
|
||||
(refs_dir / "api.md").write_text("# API Reference")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
refs = loader.load_references(skill)
|
||||
|
||||
assert "api.md" in refs
|
||||
assert "# API Reference" in refs["api.md"]
|
||||
|
||||
def test_load_assets(self):
|
||||
"""Test loading assets from skill."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create assets directory
|
||||
assets_dir = skill_dir / "assets"
|
||||
assets_dir.mkdir()
|
||||
(assets_dir / "template.json").write_text("{}")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
assets = loader.load_assets(skill)
|
||||
|
||||
assert "template.json" in assets
|
||||
# Path may be resolved differently on macOS (/var vs /private/var)
|
||||
assert assets["template.json"].endswith("test-skill/assets/template.json")
|
||||
|
||||
def test_class_method_load(self):
|
||||
"""Test convenience class method load."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
Instructions here.
|
||||
""")
|
||||
|
||||
skill = SkillLoader.load(str(skill_dir), activate=True)
|
||||
|
||||
assert skill is not None
|
||||
assert skill.is_activated
|
||||
assert "Instructions here" in skill.instructions
|
||||
|
||||
|
||||
class TestLoadedSkill:
|
||||
"""Tests for LoadedSkill dataclass."""
|
||||
|
||||
def test_metadata_property(self):
|
||||
"""Test metadata property returns SkillMetadata."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
|
||||
meta = skill.metadata
|
||||
|
||||
assert isinstance(meta, SkillMetadata)
|
||||
assert meta.name == "test-skill"
|
||||
assert meta.description == "A test skill"
|
||||
|
||||
def test_is_activated_property(self):
|
||||
"""Test is_activated property."""
|
||||
from praisonaiagents.skills.loader import SkillLoader
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
Body.
|
||||
""")
|
||||
|
||||
loader = SkillLoader()
|
||||
skill = loader.load_metadata(str(skill_dir))
|
||||
|
||||
assert skill.is_activated is False
|
||||
|
||||
loader.activate(skill)
|
||||
|
||||
assert skill.is_activated is True
|
||||
@@ -0,0 +1,254 @@
|
||||
"""Tests for Agent Skills manager."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestSkillManager:
|
||||
"""Tests for SkillManager class."""
|
||||
|
||||
def test_init_empty(self):
|
||||
"""Test initializing empty manager."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
manager = SkillManager()
|
||||
|
||||
assert len(manager) == 0
|
||||
assert manager.skill_names == []
|
||||
|
||||
def test_discover_skills(self):
|
||||
"""Test discovering skills from directory."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create a skill
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
count = manager.discover([tmpdir], include_defaults=False)
|
||||
|
||||
assert count == 1
|
||||
assert len(manager) == 1
|
||||
assert "test-skill" in manager
|
||||
|
||||
def test_add_skill(self):
|
||||
"""Test adding a single skill."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "my-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: my-skill
|
||||
description: My skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
skill = manager.add_skill(str(skill_dir))
|
||||
|
||||
assert skill is not None
|
||||
assert skill.properties.name == "my-skill"
|
||||
assert "my-skill" in manager
|
||||
|
||||
def test_get_skill(self):
|
||||
"""Test getting a skill by name."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
skill = manager.get_skill("test-skill")
|
||||
|
||||
assert skill is not None
|
||||
assert skill.properties.name == "test-skill"
|
||||
|
||||
def test_get_skill_not_found(self):
|
||||
"""Test getting nonexistent skill."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
manager = SkillManager()
|
||||
skill = manager.get_skill("nonexistent")
|
||||
|
||||
assert skill is None
|
||||
|
||||
def test_activate_by_name(self):
|
||||
"""Test activating skill by name."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
# Instructions
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
result = manager.activate_by_name("test-skill")
|
||||
|
||||
assert result is True
|
||||
skill = manager.get_skill("test-skill")
|
||||
assert skill.is_activated
|
||||
|
||||
def test_get_available_skills(self):
|
||||
"""Test getting available skills metadata."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
skills = manager.get_available_skills()
|
||||
|
||||
assert len(skills) == 1
|
||||
assert isinstance(skills[0], SkillMetadata)
|
||||
assert skills[0].name == "test-skill"
|
||||
|
||||
def test_to_prompt(self):
|
||||
"""Test generating prompt XML."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
prompt = manager.to_prompt()
|
||||
|
||||
assert "<available_skills>" in prompt
|
||||
assert "test-skill" in prompt
|
||||
assert "A test skill" in prompt
|
||||
|
||||
def test_get_instructions(self):
|
||||
"""Test getting instructions for a skill."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
|
||||
# Instructions
|
||||
|
||||
Do this and that.
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
instructions = manager.get_instructions("test-skill")
|
||||
|
||||
assert instructions is not None
|
||||
assert "Do this and that" in instructions
|
||||
|
||||
def test_load_resources(self):
|
||||
"""Test loading resources for a skill."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create scripts directory
|
||||
scripts_dir = skill_dir / "scripts"
|
||||
scripts_dir.mkdir()
|
||||
(scripts_dir / "helper.py").write_text("print('hello')")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
|
||||
result = manager.load_resources("test-skill")
|
||||
|
||||
assert result is True
|
||||
skill = manager.get_skill("test-skill")
|
||||
assert skill.resources_loaded
|
||||
|
||||
def test_clear(self):
|
||||
"""Test clearing all skills."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.add_skill(str(skill_dir))
|
||||
assert len(manager) == 1
|
||||
|
||||
manager.clear()
|
||||
|
||||
assert len(manager) == 0
|
||||
|
||||
def test_iteration(self):
|
||||
"""Test iterating over skills."""
|
||||
from praisonaiagents.skills.manager import SkillManager
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create two skills
|
||||
for name in ["skill-a", "skill-b"]:
|
||||
skill_dir = Path(tmpdir) / name
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text(f"""---
|
||||
name: {name}
|
||||
description: Skill {name}
|
||||
---
|
||||
""")
|
||||
|
||||
manager = SkillManager()
|
||||
manager.discover([tmpdir], include_defaults=False)
|
||||
|
||||
names = [s.properties.name for s in manager]
|
||||
|
||||
assert len(names) == 2
|
||||
assert "skill-a" in names
|
||||
assert "skill-b" in names
|
||||
@@ -0,0 +1,184 @@
|
||||
"""Tests for Agent Skills models."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class TestSkillProperties:
|
||||
"""Tests for SkillProperties dataclass."""
|
||||
|
||||
def test_skill_properties_required_fields(self):
|
||||
"""Test that SkillProperties requires name and description."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill for unit testing"
|
||||
)
|
||||
|
||||
assert skill.name == "test-skill"
|
||||
assert skill.description == "A test skill for unit testing"
|
||||
|
||||
def test_skill_properties_optional_fields_default_none(self):
|
||||
"""Test that optional fields default to None."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill"
|
||||
)
|
||||
|
||||
assert skill.license is None
|
||||
assert skill.compatibility is None
|
||||
assert skill.allowed_tools is None
|
||||
assert skill.path is None
|
||||
|
||||
def test_skill_properties_metadata_defaults_empty_dict(self):
|
||||
"""Test that metadata defaults to empty dict."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill"
|
||||
)
|
||||
|
||||
assert skill.metadata == {}
|
||||
|
||||
def test_skill_properties_all_fields(self):
|
||||
"""Test SkillProperties with all fields populated."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="pdf-processing",
|
||||
description="Extract text and tables from PDF files",
|
||||
license="Apache-2.0",
|
||||
compatibility="Requires pypdf package",
|
||||
allowed_tools="Read Grep Bash",
|
||||
metadata={"author": "test-org", "version": "1.0"},
|
||||
path=Path("/path/to/skill")
|
||||
)
|
||||
|
||||
assert skill.name == "pdf-processing"
|
||||
assert skill.description == "Extract text and tables from PDF files"
|
||||
assert skill.license == "Apache-2.0"
|
||||
assert skill.compatibility == "Requires pypdf package"
|
||||
assert skill.allowed_tools == "Read Grep Bash"
|
||||
assert skill.metadata == {"author": "test-org", "version": "1.0"}
|
||||
assert skill.path == Path("/path/to/skill")
|
||||
|
||||
def test_skill_properties_to_dict(self):
|
||||
"""Test to_dict method excludes None values."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
license="MIT"
|
||||
)
|
||||
|
||||
result = skill.to_dict()
|
||||
|
||||
assert result["name"] == "test-skill"
|
||||
assert result["description"] == "A test skill"
|
||||
assert result["license"] == "MIT"
|
||||
assert "compatibility" not in result
|
||||
assert "allowed-tools" not in result
|
||||
assert "metadata" not in result # Empty metadata excluded
|
||||
|
||||
def test_skill_properties_to_dict_with_metadata(self):
|
||||
"""Test to_dict includes non-empty metadata."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
metadata={"version": "1.0"}
|
||||
)
|
||||
|
||||
result = skill.to_dict()
|
||||
|
||||
assert result["metadata"] == {"version": "1.0"}
|
||||
|
||||
def test_skill_properties_to_dict_allowed_tools_key(self):
|
||||
"""Test to_dict uses 'allowed-tools' key (hyphenated)."""
|
||||
from praisonaiagents.skills.models import SkillProperties
|
||||
|
||||
skill = SkillProperties(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
allowed_tools="Read Write"
|
||||
)
|
||||
|
||||
result = skill.to_dict()
|
||||
|
||||
assert "allowed-tools" in result
|
||||
assert result["allowed-tools"] == "Read Write"
|
||||
|
||||
|
||||
class TestSkillMetadata:
|
||||
"""Tests for SkillMetadata dataclass (lightweight for system prompt)."""
|
||||
|
||||
def test_skill_metadata_required_fields(self):
|
||||
"""Test SkillMetadata with required fields."""
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
meta = SkillMetadata(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
location="/path/to/skill/SKILL.md"
|
||||
)
|
||||
|
||||
assert meta.name == "test-skill"
|
||||
assert meta.description == "A test skill"
|
||||
assert meta.location == "/path/to/skill/SKILL.md"
|
||||
|
||||
def test_skill_metadata_from_properties(self):
|
||||
"""Test creating SkillMetadata from SkillProperties."""
|
||||
from praisonaiagents.skills.models import SkillProperties, SkillMetadata
|
||||
|
||||
props = SkillProperties(
|
||||
name="pdf-skill",
|
||||
description="Process PDF files",
|
||||
path=Path("/skills/pdf-skill")
|
||||
)
|
||||
|
||||
meta = SkillMetadata.from_properties(props)
|
||||
|
||||
assert meta.name == "pdf-skill"
|
||||
assert meta.description == "Process PDF files"
|
||||
assert "/skills/pdf-skill" in meta.location
|
||||
|
||||
|
||||
class TestParseError:
|
||||
"""Tests for ParseError exception."""
|
||||
|
||||
def test_parse_error_message(self):
|
||||
"""Test ParseError stores message correctly."""
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
error = ParseError("Invalid YAML frontmatter")
|
||||
|
||||
assert str(error) == "Invalid YAML frontmatter"
|
||||
|
||||
def test_parse_error_is_exception(self):
|
||||
"""Test ParseError is an Exception subclass."""
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
assert issubclass(ParseError, Exception)
|
||||
|
||||
|
||||
class TestValidationError:
|
||||
"""Tests for ValidationError exception."""
|
||||
|
||||
def test_validation_error_message(self):
|
||||
"""Test ValidationError stores message correctly."""
|
||||
from praisonaiagents.skills.models import ValidationError
|
||||
|
||||
error = ValidationError("Missing required field: name")
|
||||
|
||||
assert str(error) == "Missing required field: name"
|
||||
|
||||
def test_validation_error_is_exception(self):
|
||||
"""Test ValidationError is an Exception subclass."""
|
||||
from praisonaiagents.skills.models import ValidationError
|
||||
|
||||
assert issubclass(ValidationError, Exception)
|
||||
@@ -0,0 +1,288 @@
|
||||
"""Tests for Agent Skills parser."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestParseFrontmatter:
|
||||
"""Tests for parse_frontmatter function."""
|
||||
|
||||
def test_parse_valid_frontmatter(self):
|
||||
"""Test parsing valid YAML frontmatter."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
|
||||
content = """---
|
||||
name: test-skill
|
||||
description: A test skill for unit testing
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
|
||||
Instructions go here.
|
||||
"""
|
||||
metadata, body = parse_frontmatter(content)
|
||||
|
||||
assert metadata["name"] == "test-skill"
|
||||
assert metadata["description"] == "A test skill for unit testing"
|
||||
assert "# Test Skill" in body
|
||||
|
||||
def test_parse_frontmatter_with_optional_fields(self):
|
||||
"""Test parsing frontmatter with all optional fields."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
|
||||
content = """---
|
||||
name: pdf-processing
|
||||
description: Extract text from PDFs
|
||||
license: Apache-2.0
|
||||
compatibility: Requires pypdf
|
||||
allowed-tools: Read Grep
|
||||
metadata:
|
||||
author: test-org
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
Body content.
|
||||
"""
|
||||
metadata, body = parse_frontmatter(content)
|
||||
|
||||
assert metadata["name"] == "pdf-processing"
|
||||
assert metadata["license"] == "Apache-2.0"
|
||||
assert metadata["compatibility"] == "Requires pypdf"
|
||||
assert metadata["allowed-tools"] == "Read Grep"
|
||||
assert metadata["metadata"]["author"] == "test-org"
|
||||
assert metadata["metadata"]["version"] == "1.0"
|
||||
|
||||
def test_parse_frontmatter_missing_opening_delimiter(self):
|
||||
"""Test error when frontmatter doesn't start with ---."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
content = """name: test-skill
|
||||
description: Missing opening delimiter
|
||||
---
|
||||
|
||||
Body content.
|
||||
"""
|
||||
try:
|
||||
parse_frontmatter(content)
|
||||
assert False, "Should have raised ParseError"
|
||||
except ParseError as e:
|
||||
assert "must start with" in str(e).lower() or "---" in str(e)
|
||||
|
||||
def test_parse_frontmatter_missing_closing_delimiter(self):
|
||||
"""Test error when frontmatter is not closed."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
content = """---
|
||||
name: test-skill
|
||||
description: Missing closing delimiter
|
||||
|
||||
Body content without closing delimiter.
|
||||
"""
|
||||
try:
|
||||
parse_frontmatter(content)
|
||||
assert False, "Should have raised ParseError"
|
||||
except ParseError as e:
|
||||
assert "closed" in str(e).lower() or "---" in str(e)
|
||||
|
||||
def test_parse_frontmatter_invalid_yaml(self):
|
||||
"""Test error when YAML is invalid."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
content = """---
|
||||
name: test-skill
|
||||
description: [invalid yaml
|
||||
- missing bracket
|
||||
---
|
||||
|
||||
Body content.
|
||||
"""
|
||||
try:
|
||||
parse_frontmatter(content)
|
||||
assert False, "Should have raised ParseError"
|
||||
except ParseError as e:
|
||||
assert "yaml" in str(e).lower() or "invalid" in str(e).lower()
|
||||
|
||||
def test_parse_frontmatter_empty_body(self):
|
||||
"""Test parsing when body is empty."""
|
||||
from praisonaiagents.skills.parser import parse_frontmatter
|
||||
|
||||
content = """---
|
||||
name: test-skill
|
||||
description: A skill with no body
|
||||
---
|
||||
"""
|
||||
metadata, body = parse_frontmatter(content)
|
||||
|
||||
assert metadata["name"] == "test-skill"
|
||||
assert body.strip() == ""
|
||||
|
||||
|
||||
class TestFindSkillMd:
|
||||
"""Tests for find_skill_md function."""
|
||||
|
||||
def test_find_skill_md_uppercase(self):
|
||||
"""Test finding SKILL.md (uppercase)."""
|
||||
from praisonaiagents.skills.parser import find_skill_md
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir)
|
||||
skill_md = skill_dir / "SKILL.md"
|
||||
skill_md.write_text("---\nname: test\ndescription: test\n---\n")
|
||||
|
||||
result = find_skill_md(skill_dir)
|
||||
|
||||
assert result is not None
|
||||
assert result.name == "SKILL.md"
|
||||
|
||||
def test_find_skill_md_lowercase(self):
|
||||
"""Test finding skill.md (lowercase) as fallback."""
|
||||
from praisonaiagents.skills.parser import find_skill_md
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir)
|
||||
skill_md = skill_dir / "skill.md"
|
||||
skill_md.write_text("---\nname: test\ndescription: test\n---\n")
|
||||
|
||||
result = find_skill_md(skill_dir)
|
||||
|
||||
assert result is not None
|
||||
# On case-insensitive filesystems (macOS), SKILL.md may match skill.md
|
||||
assert result.name.lower() == "skill.md"
|
||||
|
||||
def test_find_skill_md_prefers_uppercase(self):
|
||||
"""Test that SKILL.md is preferred over skill.md."""
|
||||
from praisonaiagents.skills.parser import find_skill_md
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir)
|
||||
(skill_dir / "SKILL.md").write_text("uppercase")
|
||||
(skill_dir / "skill.md").write_text("lowercase")
|
||||
|
||||
result = find_skill_md(skill_dir)
|
||||
|
||||
assert result is not None
|
||||
assert result.name == "SKILL.md"
|
||||
|
||||
def test_find_skill_md_not_found(self):
|
||||
"""Test returning None when no SKILL.md exists."""
|
||||
from praisonaiagents.skills.parser import find_skill_md
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir)
|
||||
|
||||
result = find_skill_md(skill_dir)
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestReadProperties:
|
||||
"""Tests for read_properties function."""
|
||||
|
||||
def test_read_properties_valid_skill(self):
|
||||
"""Test reading properties from a valid skill directory."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill for unit testing
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
""")
|
||||
|
||||
props = read_properties(skill_dir)
|
||||
|
||||
assert props.name == "test-skill"
|
||||
assert props.description == "A test skill for unit testing"
|
||||
|
||||
def test_read_properties_with_all_fields(self):
|
||||
"""Test reading properties with all optional fields."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "pdf-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: pdf-skill
|
||||
description: Process PDF files
|
||||
license: MIT
|
||||
compatibility: Requires pypdf
|
||||
allowed-tools: Read Write
|
||||
metadata:
|
||||
author: test
|
||||
version: "2.0"
|
||||
---
|
||||
|
||||
Instructions.
|
||||
""")
|
||||
|
||||
props = read_properties(skill_dir)
|
||||
|
||||
assert props.name == "pdf-skill"
|
||||
assert props.license == "MIT"
|
||||
assert props.compatibility == "Requires pypdf"
|
||||
assert props.allowed_tools == "Read Write"
|
||||
assert props.metadata["author"] == "test"
|
||||
|
||||
def test_read_properties_missing_skill_md(self):
|
||||
"""Test error when SKILL.md is missing."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
from praisonaiagents.skills.models import ParseError
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "empty-skill"
|
||||
skill_dir.mkdir()
|
||||
|
||||
try:
|
||||
read_properties(skill_dir)
|
||||
assert False, "Should have raised ParseError"
|
||||
except ParseError as e:
|
||||
assert "not found" in str(e).lower() or "skill.md" in str(e).lower()
|
||||
|
||||
def test_read_properties_missing_name(self):
|
||||
"""Test error when name field is missing."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
from praisonaiagents.skills.models import ValidationError
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "no-name"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
description: Missing name field
|
||||
---
|
||||
|
||||
Body.
|
||||
""")
|
||||
|
||||
try:
|
||||
read_properties(skill_dir)
|
||||
assert False, "Should have raised ValidationError"
|
||||
except ValidationError as e:
|
||||
assert "name" in str(e).lower()
|
||||
|
||||
def test_read_properties_missing_description(self):
|
||||
"""Test error when description field is missing."""
|
||||
from praisonaiagents.skills.parser import read_properties
|
||||
from praisonaiagents.skills.models import ValidationError
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "no-desc"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: no-desc
|
||||
---
|
||||
|
||||
Body.
|
||||
""")
|
||||
|
||||
try:
|
||||
read_properties(skill_dir)
|
||||
assert False, "Should have raised ValidationError"
|
||||
except ValidationError as e:
|
||||
assert "description" in str(e).lower()
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Tests for Agent Skills prompt generation."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestToPrompt:
|
||||
"""Tests for to_prompt function (XML generation)."""
|
||||
|
||||
def test_to_prompt_empty_list(self):
|
||||
"""Test generating prompt with no skills."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
result = to_prompt([])
|
||||
|
||||
assert "<available_skills>" in result
|
||||
assert "</available_skills>" in result
|
||||
|
||||
def test_to_prompt_single_skill(self):
|
||||
"""Test generating prompt with one skill."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill for prompt generation
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
""")
|
||||
|
||||
result = to_prompt([skill_dir])
|
||||
|
||||
assert "<available_skills>" in result
|
||||
assert "</available_skills>" in result
|
||||
assert "<skill>" in result
|
||||
assert "</skill>" in result
|
||||
assert "<name>" in result
|
||||
assert "test-skill" in result
|
||||
assert "<description>" in result
|
||||
assert "A test skill for prompt generation" in result
|
||||
assert "<location>" in result
|
||||
|
||||
def test_to_prompt_multiple_skills(self):
|
||||
"""Test generating prompt with multiple skills."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Create first skill
|
||||
skill1_dir = Path(tmpdir) / "skill-one"
|
||||
skill1_dir.mkdir()
|
||||
(skill1_dir / "SKILL.md").write_text("""---
|
||||
name: skill-one
|
||||
description: First skill
|
||||
---
|
||||
""")
|
||||
|
||||
# Create second skill
|
||||
skill2_dir = Path(tmpdir) / "skill-two"
|
||||
skill2_dir.mkdir()
|
||||
(skill2_dir / "SKILL.md").write_text("""---
|
||||
name: skill-two
|
||||
description: Second skill
|
||||
---
|
||||
""")
|
||||
|
||||
result = to_prompt([skill1_dir, skill2_dir])
|
||||
|
||||
assert result.count("<skill>") == 2
|
||||
assert result.count("</skill>") == 2
|
||||
assert "skill-one" in result
|
||||
assert "skill-two" in result
|
||||
assert "First skill" in result
|
||||
assert "Second skill" in result
|
||||
|
||||
def test_to_prompt_escapes_html(self):
|
||||
"""Test that special characters are HTML escaped."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "escape-test"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: escape-test
|
||||
description: Test <script> & "quotes" handling
|
||||
---
|
||||
""")
|
||||
|
||||
result = to_prompt([skill_dir])
|
||||
|
||||
# Should escape < > & "
|
||||
assert "<script>" not in result or "<script>" in result
|
||||
assert "&" in result or "& " not in result
|
||||
|
||||
def test_to_prompt_includes_location(self):
|
||||
"""Test that location points to SKILL.md file."""
|
||||
from praisonaiagents.skills.prompt import to_prompt
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "location-test"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: location-test
|
||||
description: Test location field
|
||||
---
|
||||
""")
|
||||
|
||||
result = to_prompt([skill_dir])
|
||||
|
||||
assert "<location>" in result
|
||||
assert "SKILL.md" in result
|
||||
assert "</location>" in result
|
||||
|
||||
|
||||
class TestGenerateSkillsXml:
|
||||
"""Tests for generate_skills_xml helper function."""
|
||||
|
||||
def test_generate_skills_xml_from_metadata_list(self):
|
||||
"""Test generating XML from SkillMetadata list."""
|
||||
from praisonaiagents.skills.prompt import generate_skills_xml
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
skills = [
|
||||
SkillMetadata(
|
||||
name="pdf-skill",
|
||||
description="Process PDF files",
|
||||
location="/path/to/pdf-skill/SKILL.md"
|
||||
),
|
||||
SkillMetadata(
|
||||
name="excel-skill",
|
||||
description="Process Excel files",
|
||||
location="/path/to/excel-skill/SKILL.md"
|
||||
)
|
||||
]
|
||||
|
||||
result = generate_skills_xml(skills)
|
||||
|
||||
assert "<available_skills>" in result
|
||||
assert "pdf-skill" in result
|
||||
assert "excel-skill" in result
|
||||
assert "Process PDF files" in result
|
||||
assert "Process Excel files" in result
|
||||
|
||||
def test_generate_skills_xml_empty_list(self):
|
||||
"""Test generating XML with empty list."""
|
||||
from praisonaiagents.skills.prompt import generate_skills_xml
|
||||
|
||||
result = generate_skills_xml([])
|
||||
|
||||
assert "<available_skills>" in result
|
||||
assert "</available_skills>" in result
|
||||
assert "<skill>" not in result
|
||||
|
||||
|
||||
class TestFormatSkillForPrompt:
|
||||
"""Tests for format_skill_for_prompt helper."""
|
||||
|
||||
def test_format_single_skill(self):
|
||||
"""Test formatting a single skill for prompt."""
|
||||
from praisonaiagents.skills.prompt import format_skill_for_prompt
|
||||
from praisonaiagents.skills.models import SkillMetadata
|
||||
|
||||
skill = SkillMetadata(
|
||||
name="test-skill",
|
||||
description="A test skill",
|
||||
location="/path/to/SKILL.md"
|
||||
)
|
||||
|
||||
result = format_skill_for_prompt(skill)
|
||||
|
||||
assert "<skill>" in result
|
||||
assert "</skill>" in result
|
||||
assert "<name>" in result
|
||||
assert "test-skill" in result
|
||||
assert "<description>" in result
|
||||
assert "A test skill" in result
|
||||
assert "<location>" in result
|
||||
assert "/path/to/SKILL.md" in result
|
||||
@@ -0,0 +1,258 @@
|
||||
"""Tests for Agent Skills validator."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
|
||||
class TestValidateName:
|
||||
"""Tests for skill name validation."""
|
||||
|
||||
def test_valid_name_simple(self):
|
||||
"""Test valid simple skill name."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("test-skill", None)
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_valid_name_with_numbers(self):
|
||||
"""Test valid name with numbers."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("skill-v2", None)
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_invalid_name_uppercase(self):
|
||||
"""Test that uppercase letters are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("Test-Skill", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("lowercase" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_starts_with_hyphen(self):
|
||||
"""Test that names starting with hyphen are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("-skill", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("hyphen" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_ends_with_hyphen(self):
|
||||
"""Test that names ending with hyphen are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("skill-", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("hyphen" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_consecutive_hyphens(self):
|
||||
"""Test that consecutive hyphens are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("test--skill", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("consecutive" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_too_long(self):
|
||||
"""Test that names over 64 characters are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
long_name = "a" * 65
|
||||
errors = _validate_name(long_name, None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("64" in e or "limit" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_name_special_characters(self):
|
||||
"""Test that special characters are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
errors = _validate_name("test_skill", None)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("invalid" in e.lower() or "character" in e.lower() for e in errors)
|
||||
|
||||
def test_name_must_match_directory(self):
|
||||
"""Test that name must match parent directory name."""
|
||||
from praisonaiagents.skills.validator import _validate_name
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "actual-name"
|
||||
skill_dir.mkdir()
|
||||
|
||||
errors = _validate_name("different-name", skill_dir)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("match" in e.lower() for e in errors)
|
||||
|
||||
|
||||
class TestValidateDescription:
|
||||
"""Tests for skill description validation."""
|
||||
|
||||
def test_valid_description(self):
|
||||
"""Test valid description."""
|
||||
from praisonaiagents.skills.validator import _validate_description
|
||||
|
||||
errors = _validate_description("A helpful skill that does things")
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_invalid_description_empty(self):
|
||||
"""Test that empty description is rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_description
|
||||
|
||||
errors = _validate_description("")
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("empty" in e.lower() or "non-empty" in e.lower() for e in errors)
|
||||
|
||||
def test_invalid_description_too_long(self):
|
||||
"""Test that descriptions over 1024 characters are rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_description
|
||||
|
||||
long_desc = "a" * 1025
|
||||
errors = _validate_description(long_desc)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("1024" in e or "limit" in e.lower() for e in errors)
|
||||
|
||||
|
||||
class TestValidateCompatibility:
|
||||
"""Tests for skill compatibility validation."""
|
||||
|
||||
def test_valid_compatibility(self):
|
||||
"""Test valid compatibility string."""
|
||||
from praisonaiagents.skills.validator import _validate_compatibility
|
||||
|
||||
errors = _validate_compatibility("Requires Python 3.11+")
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_invalid_compatibility_too_long(self):
|
||||
"""Test that compatibility over 500 characters is rejected."""
|
||||
from praisonaiagents.skills.validator import _validate_compatibility
|
||||
|
||||
long_compat = "a" * 501
|
||||
errors = _validate_compatibility(long_compat)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("500" in e or "limit" in e.lower() for e in errors)
|
||||
|
||||
|
||||
class TestValidateMetadata:
|
||||
"""Tests for full metadata validation."""
|
||||
|
||||
def test_valid_metadata(self):
|
||||
"""Test valid complete metadata."""
|
||||
from praisonaiagents.skills.validator import validate_metadata
|
||||
|
||||
metadata = {
|
||||
"name": "test-skill",
|
||||
"description": "A test skill"
|
||||
}
|
||||
|
||||
errors = validate_metadata(metadata)
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_missing_name(self):
|
||||
"""Test error when name is missing."""
|
||||
from praisonaiagents.skills.validator import validate_metadata
|
||||
|
||||
metadata = {
|
||||
"description": "A test skill"
|
||||
}
|
||||
|
||||
errors = validate_metadata(metadata)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("name" in e.lower() for e in errors)
|
||||
|
||||
def test_missing_description(self):
|
||||
"""Test error when description is missing."""
|
||||
from praisonaiagents.skills.validator import validate_metadata
|
||||
|
||||
metadata = {
|
||||
"name": "test-skill"
|
||||
}
|
||||
|
||||
errors = validate_metadata(metadata)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("description" in e.lower() for e in errors)
|
||||
|
||||
def test_unexpected_fields_rejected(self):
|
||||
"""Test that unexpected fields are rejected."""
|
||||
from praisonaiagents.skills.validator import validate_metadata
|
||||
|
||||
metadata = {
|
||||
"name": "test-skill",
|
||||
"description": "A test skill",
|
||||
"unknown-field": "value"
|
||||
}
|
||||
|
||||
errors = validate_metadata(metadata)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("unexpected" in e.lower() or "unknown-field" in e for e in errors)
|
||||
|
||||
|
||||
class TestValidate:
|
||||
"""Tests for full skill directory validation."""
|
||||
|
||||
def test_validate_valid_skill(self):
|
||||
"""Test validating a valid skill directory."""
|
||||
from praisonaiagents.skills.validator import validate
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
(skill_dir / "SKILL.md").write_text("""---
|
||||
name: test-skill
|
||||
description: A test skill for validation
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
""")
|
||||
|
||||
errors = validate(skill_dir)
|
||||
|
||||
assert errors == []
|
||||
|
||||
def test_validate_missing_skill_md(self):
|
||||
"""Test error when SKILL.md is missing."""
|
||||
from praisonaiagents.skills.validator import validate
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_dir = Path(tmpdir) / "empty-skill"
|
||||
skill_dir.mkdir()
|
||||
|
||||
errors = validate(skill_dir)
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("skill.md" in e.lower() for e in errors)
|
||||
|
||||
def test_validate_nonexistent_directory(self):
|
||||
"""Test error when directory doesn't exist."""
|
||||
from praisonaiagents.skills.validator import validate
|
||||
|
||||
errors = validate(Path("/nonexistent/path"))
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("exist" in e.lower() for e in errors)
|
||||
|
||||
def test_validate_not_a_directory(self):
|
||||
"""Test error when path is not a directory."""
|
||||
from praisonaiagents.skills.validator import validate
|
||||
|
||||
with tempfile.NamedTemporaryFile() as tmpfile:
|
||||
errors = validate(Path(tmpfile.name))
|
||||
|
||||
assert len(errors) > 0
|
||||
assert any("directory" in e.lower() for e in errors)
|
||||
Generated
+1
-1
@@ -2422,7 +2422,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "praisonaiagents"
|
||||
version = "0.1.17"
|
||||
version = "0.1.25"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "aiohttp" },
|
||||
|
||||
+245
-187
@@ -171,12 +171,13 @@ npm install praisonai
|
||||
|
||||
| Feature | Code | Docs |
|
||||
|---------|:----:|:----:|
|
||||
| 🚀 Single Agent | [Example](examples/python/agents/single-agent.py) | [📖](https://docs.praison.ai/agents/single) |
|
||||
| 🤝 Multi Agents | [Example](examples/python/general/mini_agents_example.py) | [📖](https://docs.praison.ai/concepts/agents) |
|
||||
| 🤖 Auto Agents | [Example](examples/python/general/auto_agents_example.py) | [📖](https://docs.praison.ai/features/autoagents) |
|
||||
| 🔄 Self Reflection AI Agents | [Example](examples/python/concepts/self-reflection-details.py) | [📖](https://docs.praison.ai/features/selfreflection) |
|
||||
| 🧠 Reasoning AI Agents | [Example](examples/python/concepts/reasoning-extraction.py) | [📖](https://docs.praison.ai/features/reasoning) |
|
||||
| 👁️ Multi Modal AI Agents | [Example](examples/python/general/multimodal.py) | [📖](https://docs.praison.ai/features/multimodal) |
|
||||
| **🤖 Core Agents** | | |
|
||||
| ↳ Single Agent | [Example](examples/python/agents/single-agent.py) | [📖](https://docs.praison.ai/agents/single) |
|
||||
| ↳ Multi Agents | [Example](examples/python/general/mini_agents_example.py) | [📖](https://docs.praison.ai/concepts/agents) |
|
||||
| ↳ Auto Agents | [Example](examples/python/general/auto_agents_example.py) | [📖](https://docs.praison.ai/features/autoagents) |
|
||||
| ↳ Self Reflection AI Agents | [Example](examples/python/concepts/self-reflection-details.py) | [📖](https://docs.praison.ai/features/selfreflection) |
|
||||
| ↳ Reasoning AI Agents | [Example](examples/python/concepts/reasoning-extraction.py) | [📖](https://docs.praison.ai/features/reasoning) |
|
||||
| ↳ Multi Modal AI Agents | [Example](examples/python/general/multimodal.py) | [📖](https://docs.praison.ai/features/multimodal) |
|
||||
| **🔄 Workflows** | | |
|
||||
| ↳ Simple Workflow | [Example](examples/python/workflows/simple_workflow.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow with Agents | [Example](examples/python/workflows/workflow_with_agents.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
@@ -188,10 +189,7 @@ npm install praisonai
|
||||
| ↳ Workflow Branching | [Example](examples/python/workflows/workflow_branching.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow Early Stop | [Example](examples/python/workflows/workflow_early_stop.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| ↳ Workflow Checkpoints | [Example](examples/python/workflows/workflow_checkpoints.py) | [📖](https://docs.praison.ai/features/workflows) |
|
||||
| 📚 Add Custom Knowledge | [Example](examples/python/concepts/knowledge-agents.py) | [📖](https://docs.praison.ai/features/knowledge) |
|
||||
| 🧠 Memory (Short & Long Term) | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| 📄 Chat with PDF Agents | [Example](examples/python/concepts/chat-with-pdf.py) | [📖](https://docs.praison.ai/features/chat-with-pdf) |
|
||||
| **💻 Code** | | |
|
||||
| **💻 Code & Development** | | |
|
||||
| ↳ Code Interpreter Agents | [Example](examples/python/agents/code-agent.py) | [📖](https://docs.praison.ai/features/codeagent) |
|
||||
| ↳ AI Code Editing Tools | [Example](examples/python/code/code_editing_example.py) | [📖](https://docs.praison.ai/code/editing) |
|
||||
| ↳ External Agents (All) | [Example](examples/python/code/external_agents_example.py) | [📖](https://docs.praison.ai/code/external-agents) |
|
||||
@@ -199,65 +197,78 @@ npm install praisonai
|
||||
| ↳ Gemini CLI | [Example](examples/python/code/gemini_cli_example.py) | [📖](https://docs.praison.ai/code/gemini-cli) |
|
||||
| ↳ Codex CLI | [Example](examples/python/code/codex_cli_example.py) | [📖](https://docs.praison.ai/code/codex-cli) |
|
||||
| ↳ Cursor CLI | [Example](examples/python/code/cursor_cli_example.py) | [📖](https://docs.praison.ai/code/cursor-cli) |
|
||||
| 📚 RAG Agents | [Example](examples/python/concepts/rag-agents.py) | [📖](https://docs.praison.ai/features/rag) |
|
||||
| 🤔 Async & Parallel Processing | [Example](examples/python/general/async_example.py) | [📖](https://docs.praison.ai/features/async) |
|
||||
| 🔢 Math Agents | [Example](examples/python/agents/math-agent.py) | [📖](https://docs.praison.ai/features/mathagent) |
|
||||
| 🎯 Structured Output Agents | [Example](examples/python/general/structured_agents_example.py) | [📖](https://docs.praison.ai/features/structured) |
|
||||
| 🔗 LangChain Integrated Agents | [Example](examples/python/general/langchain_example.py) | [📖](https://docs.praison.ai/features/langchain) |
|
||||
| 📞 Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
|
||||
| 🛠️ 100+ Custom Tools | [Example](examples/python/general/tools_example.py) | [📖](https://docs.praison.ai/tools/tools) |
|
||||
| 📄 YAML Configuration | [Example](examples/cookbooks/yaml/secondary_market_research_agents.yaml) | [📖](https://docs.praison.ai/developers/agents-playbook) |
|
||||
| 💯 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
|
||||
| 🔬 Deep Research Agents | [Example](examples/python/agents/research-agent.py) | [📖](https://docs.praison.ai/agents/deep-research) |
|
||||
| 🔄 Query Rewriter Agent | [Example](#5-query-rewriter-agent) | [📖](https://docs.praison.ai/agents/query-rewriter) |
|
||||
| 🌐 Native Web Search | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/agents/websearch) |
|
||||
| 📥 Web Fetch (Anthropic) | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| 💾 Prompt Caching | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| 🧠 Claude Memory Tool | [Example](#claude-memory-tool-cli) | [📖](https://docs.praison.ai/features/claude-memory-tool) |
|
||||
| 💾 File-Based Memory | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| 🔍 Built-in Search Tools | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/tools/tavily) |
|
||||
| 🔎 Unified Web Search | [Example](src/praisonai-agents/examples/web_search_example.py) | [📖](https://docs.praison.ai/tools/web-search) |
|
||||
| 📋 Planning Mode | [Example](examples/python/agents/planning-agent.py) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🔧 Planning Tools | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🧠 Planning Reasoning | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| 🔌 MCP Transports | [Example](examples/python/mcp/mcp-transports-overview.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| 🌐 WebSocket MCP | [Example](examples/python/mcp/websocket-mcp.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| 🔐 MCP Security | [Example](examples/python/mcp/mcp-security.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| 🔄 MCP Resumability | [Example](examples/python/mcp/mcp-resumability.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ⚡ Fast Context | [Example](examples/context/00_agent_fast_context_basic.py) | [📖](https://docs.praison.ai/features/fast-context) |
|
||||
| 🖼️ Image Generation Agent | [Example](examples/python/image/image-agent.py) | [📖](https://docs.praison.ai/features/image-generation) |
|
||||
| 📷 Image to Text Agent | [Example](examples/python/agents/image-to-text-agent.py) | [📖](https://docs.praison.ai/agents/image-to-text) |
|
||||
| 🎬 Video Agent | [Example](examples/python/agents/video-agent.py) | [📖](https://docs.praison.ai/agents/video) |
|
||||
| 📊 Data Analyst Agent | [Example](examples/python/agents/data-analyst-agent.py) | [📖](https://docs.praison.ai/agents/data-analyst) |
|
||||
| 💰 Finance Agent | [Example](examples/python/agents/finance-agent.py) | [📖](https://docs.praison.ai/agents/finance) |
|
||||
| 🛒 Shopping Agent | [Example](examples/python/agents/shopping-agent.py) | [📖](https://docs.praison.ai/agents/shopping) |
|
||||
| ⭐ Recommendation Agent | [Example](examples/python/agents/recommendation-agent.py) | [📖](https://docs.praison.ai/agents/recommendation) |
|
||||
| 📖 Wikipedia Agent | [Example](examples/python/agents/wikipedia-agent.py) | [📖](https://docs.praison.ai/agents/wikipedia) |
|
||||
| 💻 Programming Agent | [Example](examples/python/agents/programming-agent.py) | [📖](https://docs.praison.ai/agents/programming) |
|
||||
| 📝 Markdown Agent | [Example](examples/python/agents/markdown-agent.py) | [📖](https://docs.praison.ai/agents/markdown) |
|
||||
| 📝 Prompt Expander Agent | [Example](#prompt-expansion) | [📖](https://docs.praison.ai/agents/prompt-expander) |
|
||||
| 🔀 Model Router | [Example](examples/python/agents/router-agent-cost-optimization.py) | [📖](https://docs.praison.ai/features/model-router) |
|
||||
| ⛓️ Prompt Chaining | [Example](examples/python/general/prompt_chaining.py) | [📖](https://docs.praison.ai/features/promptchaining) |
|
||||
| 🔍 Evaluator Optimiser | [Example](examples/python/general/evaluator-optimiser.py) | [📖](https://docs.praison.ai/features/evaluator-optimiser) |
|
||||
| 👷 Orchestrator Workers | [Example](examples/python/general/orchestrator-workers.py) | [📖](https://docs.praison.ai/features/orchestrator-worker) |
|
||||
| ⚡ Parallelisation | [Example](examples/python/general/parallelisation.py) | [📖](https://docs.praison.ai/features/parallelisation) |
|
||||
| 🔁 Repetitive Agents | [Example](examples/python/concepts/repetitive-agents.py) | [📖](https://docs.praison.ai/features/repetitive) |
|
||||
| 🤝 Agent Handoffs | [Example](examples/python/handoff/handoff_basic.py) | [📖](https://docs.praison.ai/features/handoffs) |
|
||||
| 🛡️ Guardrails | [Example](examples/python/guardrails/comprehensive-guardrails-example.py) | [📖](https://docs.praison.ai/features/guardrails) |
|
||||
| 💬 Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
|
||||
| ✅ Human Approval | [Example](examples/python/general/human_approval_example.py) | [📖](https://docs.praison.ai/features/approval) |
|
||||
| 🔄 Stateful Agents | [Example](examples/python/stateful/workflow-state-example.py) | [📖](https://docs.praison.ai/features/stateful-agents) |
|
||||
| 🤖 Autonomous Workflow | [Example](examples/python/general/autonomous-agent.py) | [📖](https://docs.praison.ai/features/autonomous-workflow) |
|
||||
| 📜 Rules & Instructions | [Example](#6-rules--instructions) | [📖](https://docs.praison.ai/features/rules) |
|
||||
| 🪝 Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
|
||||
| 📈 Telemetry | [Example](examples/python/telemetry/production-telemetry-example.py) | [📖](https://docs.praison.ai/features/telemetry) |
|
||||
| 📹 Camera Integration | [Example](examples/python/camera/) | [📖](https://docs.praison.ai/features/camera-integration) |
|
||||
| 📄 Project Docs (.praison/docs/) | [Example](#docs-cli) | [📖](https://docs.praison.ai/docs/cli/docs) |
|
||||
| 🔌 MCP Config Management | [Example](#mcp-config-cli) | [📖](https://docs.praison.ai/docs/cli/mcp) |
|
||||
| 💬 AI Commit Messages | [Example](#ai-commit-cli) | [📖](https://docs.praison.ai/docs/cli/commit) |
|
||||
| @ @Mentions in Prompts | [Example](#mentions-in-prompts) | [📖](https://docs.praison.ai/docs/cli/mentions) |
|
||||
| 💾 Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| 📜 History in Context | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| **🧠 Memory & Knowledge** | | |
|
||||
| ↳ Memory (Short & Long Term) | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| ↳ File-Based Memory | [Example](examples/python/general/memory_example.py) | [📖](https://docs.praison.ai/concepts/memory) |
|
||||
| ↳ Claude Memory Tool | [Example](#claude-memory-tool-cli) | [📖](https://docs.praison.ai/features/claude-memory-tool) |
|
||||
| ↳ Add Custom Knowledge | [Example](examples/python/concepts/knowledge-agents.py) | [📖](https://docs.praison.ai/features/knowledge) |
|
||||
| ↳ RAG Agents | [Example](examples/python/concepts/rag-agents.py) | [📖](https://docs.praison.ai/features/rag) |
|
||||
| ↳ Chat with PDF Agents | [Example](examples/python/concepts/chat-with-pdf.py) | [📖](https://docs.praison.ai/features/chat-with-pdf) |
|
||||
| **🔬 Research & Intelligence** | | |
|
||||
| ↳ Deep Research Agents | [Example](examples/python/agents/research-agent.py) | [📖](https://docs.praison.ai/agents/deep-research) |
|
||||
| ↳ Query Rewriter Agent | [Example](#5-query-rewriter-agent) | [📖](https://docs.praison.ai/agents/query-rewriter) |
|
||||
| ↳ Native Web Search | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/agents/websearch) |
|
||||
| ↳ Built-in Search Tools | [Example](examples/python/agents/websearch-agent.py) | [📖](https://docs.praison.ai/tools/tavily) |
|
||||
| ↳ Unified Web Search | [Example](src/praisonai-agents/examples/web_search_example.py) | [📖](https://docs.praison.ai/tools/web-search) |
|
||||
| ↳ Web Fetch (Anthropic) | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| **📋 Planning & Execution** | | |
|
||||
| ↳ Planning Mode | [Example](examples/python/agents/planning-agent.py) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Planning Tools | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Planning Reasoning | [Example](#3-agent-with-planning-mode) | [📖](https://docs.praison.ai/features/planning-mode) |
|
||||
| ↳ Prompt Chaining | [Example](examples/python/general/prompt_chaining.py) | [📖](https://docs.praison.ai/features/promptchaining) |
|
||||
| ↳ Evaluator Optimiser | [Example](examples/python/general/evaluator-optimiser.py) | [📖](https://docs.praison.ai/features/evaluator-optimiser) |
|
||||
| ↳ Orchestrator Workers | [Example](examples/python/general/orchestrator-workers.py) | [📖](https://docs.praison.ai/features/orchestrator-worker) |
|
||||
| **👥 Specialized Agents** | | |
|
||||
| ↳ Data Analyst Agent | [Example](examples/python/agents/data-analyst-agent.py) | [📖](https://docs.praison.ai/agents/data-analyst) |
|
||||
| ↳ Finance Agent | [Example](examples/python/agents/finance-agent.py) | [📖](https://docs.praison.ai/agents/finance) |
|
||||
| ↳ Shopping Agent | [Example](examples/python/agents/shopping-agent.py) | [📖](https://docs.praison.ai/agents/shopping) |
|
||||
| ↳ Recommendation Agent | [Example](examples/python/agents/recommendation-agent.py) | [📖](https://docs.praison.ai/agents/recommendation) |
|
||||
| ↳ Wikipedia Agent | [Example](examples/python/agents/wikipedia-agent.py) | [📖](https://docs.praison.ai/agents/wikipedia) |
|
||||
| ↳ Programming Agent | [Example](examples/python/agents/programming-agent.py) | [📖](https://docs.praison.ai/agents/programming) |
|
||||
| ↳ Math Agents | [Example](examples/python/agents/math-agent.py) | [📖](https://docs.praison.ai/features/mathagent) |
|
||||
| ↳ Markdown Agent | [Example](examples/python/agents/markdown-agent.py) | [📖](https://docs.praison.ai/agents/markdown) |
|
||||
| ↳ Prompt Expander Agent | [Example](#prompt-expansion) | [📖](https://docs.praison.ai/agents/prompt-expander) |
|
||||
| **🎨 Media & Multimodal** | | |
|
||||
| ↳ Image Generation Agent | [Example](examples/python/image/image-agent.py) | [📖](https://docs.praison.ai/features/image-generation) |
|
||||
| ↳ Image to Text Agent | [Example](examples/python/agents/image-to-text-agent.py) | [📖](https://docs.praison.ai/agents/image-to-text) |
|
||||
| ↳ Video Agent | [Example](examples/python/agents/video-agent.py) | [📖](https://docs.praison.ai/agents/video) |
|
||||
| ↳ Camera Integration | [Example](examples/python/camera/) | [📖](https://docs.praison.ai/features/camera-integration) |
|
||||
| **🔌 Protocols & Integration** | | |
|
||||
| ↳ MCP Transports | [Example](examples/python/mcp/mcp-transports-overview.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| ↳ WebSocket MCP | [Example](examples/python/mcp/websocket-mcp.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ↳ MCP Security | [Example](examples/python/mcp/mcp-security.py) | [📖](https://docs.praison.ai/mcp/transports) |
|
||||
| ↳ MCP Resumability | [Example](examples/python/mcp/mcp-resumability.py) | [📖](https://docs.praison.ai/mcp/sse-transport) |
|
||||
| ↳ MCP Config Management | [Example](#mcp-config-cli) | [📖](https://docs.praison.ai/docs/cli/mcp) |
|
||||
| ↳ LangChain Integrated Agents | [Example](examples/python/general/langchain_example.py) | [📖](https://docs.praison.ai/features/langchain) |
|
||||
| **🛡️ Safety & Control** | | |
|
||||
| ↳ Guardrails | [Example](examples/python/guardrails/comprehensive-guardrails-example.py) | [📖](https://docs.praison.ai/features/guardrails) |
|
||||
| ↳ Human Approval | [Example](examples/python/general/human_approval_example.py) | [📖](https://docs.praison.ai/features/approval) |
|
||||
| ↳ Rules & Instructions | [Example](#6-rules--instructions) | [📖](https://docs.praison.ai/features/rules) |
|
||||
| **⚙️ Advanced Features** | | |
|
||||
| ↳ Async & Parallel Processing | [Example](examples/python/general/async_example.py) | [📖](https://docs.praison.ai/features/async) |
|
||||
| ↳ Parallelisation | [Example](examples/python/general/parallelisation.py) | [📖](https://docs.praison.ai/features/parallelisation) |
|
||||
| ↳ Repetitive Agents | [Example](examples/python/concepts/repetitive-agents.py) | [📖](https://docs.praison.ai/features/repetitive) |
|
||||
| ↳ Agent Handoffs | [Example](examples/python/handoff/handoff_basic.py) | [📖](https://docs.praison.ai/features/handoffs) |
|
||||
| ↳ Stateful Agents | [Example](examples/python/stateful/workflow-state-example.py) | [📖](https://docs.praison.ai/features/stateful-agents) |
|
||||
| ↳ Autonomous Workflow | [Example](examples/python/general/autonomous-agent.py) | [📖](https://docs.praison.ai/features/autonomous-workflow) |
|
||||
| ↳ Structured Output Agents | [Example](examples/python/general/structured_agents_example.py) | [📖](https://docs.praison.ai/features/structured) |
|
||||
| ↳ Model Router | [Example](examples/python/agents/router-agent-cost-optimization.py) | [📖](https://docs.praison.ai/features/model-router) |
|
||||
| ↳ Prompt Caching | [Example](#web-search-web-fetch--prompt-caching) | [📖](https://docs.praison.ai/features/model-capabilities) |
|
||||
| ↳ Fast Context | [Example](examples/context/00_agent_fast_context_basic.py) | [📖](https://docs.praison.ai/features/fast-context) |
|
||||
| **🛠️ Tools & Configuration** | | |
|
||||
| ↳ 100+ Custom Tools | [Example](examples/python/general/tools_example.py) | [📖](https://docs.praison.ai/tools/tools) |
|
||||
| ↳ YAML Configuration | [Example](examples/cookbooks/yaml/secondary_market_research_agents.yaml) | [📖](https://docs.praison.ai/developers/agents-playbook) |
|
||||
| ↳ 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
|
||||
| ↳ Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
|
||||
| ↳ Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
|
||||
| **📊 Monitoring & Management** | | |
|
||||
| ↳ Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
|
||||
| ↳ Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| ↳ History in Context | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
| ↳ Telemetry | [Example](examples/python/telemetry/production-telemetry-example.py) | [📖](https://docs.praison.ai/features/telemetry) |
|
||||
| ↳ Project Docs (.praison/docs/) | [Example](#docs-cli) | [📖](https://docs.praison.ai/docs/cli/docs) |
|
||||
| ↳ AI Commit Messages | [Example](#ai-commit-cli) | [📖](https://docs.praison.ai/docs/cli/commit) |
|
||||
| ↳ @Mentions in Prompts | [Example](#mentions-in-prompts) | [📖](https://docs.praison.ai/docs/cli/mentions) |
|
||||
| **🖥️ CLI Features** | | |
|
||||
| ↳ Slash Commands | [Example](examples/python/cli/slash_commands_example.py) | [📖](https://docs.praison.ai/docs/cli/slash-commands) |
|
||||
| ↳ Autonomy Modes | [Example](examples/python/cli/autonomy_modes_example.py) | [📖](https://docs.praison.ai/docs/cli/autonomy-modes) |
|
||||
@@ -266,6 +277,23 @@ npm install praisonai
|
||||
| ↳ Interactive TUI | [Example](examples/python/cli/interactive_tui_example.py) | [📖](https://docs.praison.ai/docs/cli/interactive-tui) |
|
||||
| ↳ Git Integration | [Example](examples/python/cli/git_integration_example.py) | [📖](https://docs.praison.ai/docs/cli/git-integration) |
|
||||
| ↳ Sandbox Execution | [Example](examples/python/cli/sandbox_execution_example.py) | [📖](https://docs.praison.ai/docs/cli/sandbox-execution) |
|
||||
| ↳ CLI Compare | [Example](examples/compare/cli_compare_basic.py) | [📖](https://docs.praison.ai/docs/cli/compare) |
|
||||
| ↳ Profile/Benchmark | [Example](#profile-benchmark) | [📖](https://docs.praison.ai/docs/cli/profile) |
|
||||
| ↳ Auto Mode | [Example](#auto-mode) | [📖](https://docs.praison.ai/docs/cli/auto) |
|
||||
| ↳ Init | [Example](#init) | [📖](https://docs.praison.ai/docs/cli/init) |
|
||||
| ↳ File Input | [Example](#file-input) | [📖](https://docs.praison.ai/docs/cli/file-input) |
|
||||
| ↳ Final Agent | [Example](#final-agent) | [📖](https://docs.praison.ai/docs/cli/final-agent) |
|
||||
| ↳ Max Tokens | [Example](#max-tokens) | [📖](https://docs.praison.ai/docs/cli/max-tokens) |
|
||||
| **🧪 Evaluation** | | |
|
||||
| ↳ Accuracy Evaluation | [Example](examples/eval/accuracy_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Performance Evaluation | [Example](examples/eval/performance_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Reliability Evaluation | [Example](examples/eval/reliability_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| ↳ Criteria Evaluation | [Example](examples/eval/criteria_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
|
||||
| **🎯 Agent Skills** | | |
|
||||
| ↳ Skills Management | [Example](examples/skills/basic_skill_usage.py) | [📖](https://docs.praison.ai/features/skills) |
|
||||
| ↳ Custom Skills | [Example](examples/skills/custom_skill_example.py) | [📖](https://docs.praison.ai/features/skills) |
|
||||
| **⏰ 24/7 Scheduling** | | |
|
||||
| ↳ Agent Scheduler | [Example](examples/python/scheduled_agents/news_checker_live.py) | [📖](https://docs.praison.ai/docs/cli/scheduler) |
|
||||
|
||||
---
|
||||
|
||||
@@ -959,6 +987,124 @@ Run with the same simple command:
|
||||
praisonai agents.yaml
|
||||
```
|
||||
|
||||
### 12. MCP (Model Context Protocol)
|
||||
|
||||
PraisonAI supports MCP Protocol Revision 2025-11-25 with multiple transports.
|
||||
|
||||
#### MCP Client (Consume MCP Servers)
|
||||
```python
|
||||
from praisonaiagents import Agent, MCP
|
||||
|
||||
# stdio - Local NPX/Python servers
|
||||
agent = Agent(tools=MCP("npx @modelcontextprotocol/server-memory"))
|
||||
|
||||
# Streamable HTTP - Production servers
|
||||
agent = Agent(tools=MCP("https://api.example.com/mcp"))
|
||||
|
||||
# WebSocket - Real-time bidirectional
|
||||
agent = Agent(tools=MCP("wss://api.example.com/mcp", auth_token="token"))
|
||||
|
||||
# SSE (Legacy) - Backward compatibility
|
||||
agent = Agent(tools=MCP("http://localhost:8080/sse"))
|
||||
|
||||
# With environment variables
|
||||
agent = Agent(
|
||||
tools=MCP(
|
||||
command="npx",
|
||||
args=["-y", "@modelcontextprotocol/server-brave-search"],
|
||||
env={"BRAVE_API_KEY": "your-key"}
|
||||
)
|
||||
)
|
||||
|
||||
# Multiple MCP servers + regular functions
|
||||
def my_custom_tool(query: str) -> str:
|
||||
"""Custom tool function."""
|
||||
return f"Result: {query}"
|
||||
|
||||
agent = Agent(
|
||||
name="MultiToolAgent",
|
||||
instructions="Agent with multiple MCP servers",
|
||||
tools=[
|
||||
MCP("uvx mcp-server-time"), # Time tools
|
||||
MCP("npx @modelcontextprotocol/server-memory"), # Memory tools
|
||||
my_custom_tool # Regular function
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
#### MCP Server (Expose Tools as MCP Server)
|
||||
|
||||
Expose your Python functions as MCP tools for Claude Desktop, Cursor, and other MCP clients:
|
||||
|
||||
```python
|
||||
from praisonaiagents.mcp import ToolsMCPServer
|
||||
|
||||
def search_web(query: str, max_results: int = 5) -> dict:
|
||||
"""Search the web for information."""
|
||||
return {"results": [f"Result for {query}"]}
|
||||
|
||||
def calculate(expression: str) -> dict:
|
||||
"""Evaluate a mathematical expression."""
|
||||
return {"result": eval(expression)}
|
||||
|
||||
# Create and run MCP server
|
||||
server = ToolsMCPServer(name="my-tools")
|
||||
server.register_tools([search_web, calculate])
|
||||
server.run() # stdio for Claude Desktop
|
||||
# server.run_sse(host="0.0.0.0", port=8080) # SSE for web clients
|
||||
```
|
||||
|
||||
#### MCP Features
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| Session Management | Automatic Mcp-Session-Id handling |
|
||||
| Protocol Versioning | Mcp-Protocol-Version header |
|
||||
| Resumability | SSE stream recovery via Last-Event-ID |
|
||||
| Security | Origin validation, DNS rebinding prevention |
|
||||
| WebSocket | Auto-reconnect with exponential backoff |
|
||||
|
||||
### 13. A2A (Agent2Agent Protocol)
|
||||
|
||||
PraisonAI supports the [A2A Protocol](https://a2a-protocol.org) for agent-to-agent communication, enabling your agents to be discovered and collaborate with other AI agents.
|
||||
|
||||
#### A2A Server (Expose Agent as A2A Server)
|
||||
```python
|
||||
from praisonaiagents import Agent, A2A
|
||||
from fastapi import FastAPI
|
||||
|
||||
# Create an agent with tools
|
||||
def search_web(query: str) -> str:
|
||||
"""Search the web for information."""
|
||||
return f"Results for: {query}"
|
||||
|
||||
agent = Agent(
|
||||
name="Research Assistant",
|
||||
role="Research Analyst",
|
||||
goal="Help users research topics",
|
||||
tools=[search_web]
|
||||
)
|
||||
|
||||
# Expose as A2A Server
|
||||
a2a = A2A(agent=agent, url="http://localhost:8000/a2a")
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(a2a.get_router())
|
||||
|
||||
# Run: uvicorn app:app --reload
|
||||
# Agent Card: GET /.well-known/agent.json
|
||||
# Status: GET /status
|
||||
```
|
||||
|
||||
#### A2A Features
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| Agent Card | JSON metadata for agent discovery |
|
||||
| Skills Extraction | Auto-generate skills from tools |
|
||||
| Task Management | Stateful task lifecycle |
|
||||
| Streaming | SSE streaming for real-time updates |
|
||||
|
||||
> **Documentation**: [docs.praison.ai/a2a](https://docs.praison.ai/a2a) | **Examples**: [examples/python/a2a](https://github.com/MervinPraison/PraisonAI/tree/main/examples/python/a2a)
|
||||
|
||||
---
|
||||
|
||||
## 🎯 CLI / No-Code Interface
|
||||
@@ -1058,6 +1204,20 @@ praisonai "Complex analysis task" --planning --planning-reasoning
|
||||
praisonai "Task" --planning --auto-approve-plan
|
||||
```
|
||||
|
||||
### Tool Approval CLI:
|
||||
```bash
|
||||
# Auto-approve ALL tool executions (use with caution!)
|
||||
praisonai "run ls command" --trust
|
||||
|
||||
# Auto-approve tools up to a risk level (prompt for higher)
|
||||
# Levels: low, medium, high, critical
|
||||
praisonai "write to file" --approve-level high # Prompts for critical tools only
|
||||
praisonai "task" --approve-level medium # Prompts for high and critical
|
||||
|
||||
# Default behavior (no flags): prompts for all dangerous tools
|
||||
praisonai "run shell command" # Will prompt for approval
|
||||
```
|
||||
|
||||
### Memory CLI:
|
||||
```bash
|
||||
# Enable memory for agent (persists across sessions)
|
||||
@@ -1228,6 +1388,21 @@ praisonai "Analyze this data" --metrics
|
||||
praisonai "Complex task" --metrics --planning
|
||||
```
|
||||
|
||||
### Scheduler CLI:
|
||||
|
||||
```bash
|
||||
praisonai schedule start <name> "task" --interval hourly
|
||||
praisonai schedule list
|
||||
praisonai schedule logs <name> [--follow]
|
||||
praisonai schedule stop <name>
|
||||
praisonai schedule restart <name>
|
||||
praisonai schedule delete <name>
|
||||
praisonai schedule describe <name>
|
||||
praisonai schedule save <name> [file.yaml]
|
||||
praisonai schedule "task" --interval hourly # foreground mode
|
||||
praisonai schedule agents.yaml # foreground mode
|
||||
```
|
||||
|
||||
### Image Processing CLI:
|
||||
```bash
|
||||
# Process images with vision-based tasks
|
||||
@@ -1679,124 +1854,6 @@ agent = Agent(
|
||||
| Web Fetch | Anthropic |
|
||||
| Prompt Caching | OpenAI (auto), Anthropic, Bedrock, Deepseek |
|
||||
|
||||
## MCP (Model Context Protocol)
|
||||
|
||||
PraisonAI supports MCP Protocol Revision 2025-11-25 with multiple transports.
|
||||
|
||||
### MCP Client (Consume MCP Servers)
|
||||
```python
|
||||
from praisonaiagents import Agent, MCP
|
||||
|
||||
# stdio - Local NPX/Python servers
|
||||
agent = Agent(tools=MCP("npx @modelcontextprotocol/server-memory"))
|
||||
|
||||
# Streamable HTTP - Production servers
|
||||
agent = Agent(tools=MCP("https://api.example.com/mcp"))
|
||||
|
||||
# WebSocket - Real-time bidirectional
|
||||
agent = Agent(tools=MCP("wss://api.example.com/mcp", auth_token="token"))
|
||||
|
||||
# SSE (Legacy) - Backward compatibility
|
||||
agent = Agent(tools=MCP("http://localhost:8080/sse"))
|
||||
|
||||
# With environment variables
|
||||
agent = Agent(
|
||||
tools=MCP(
|
||||
command="npx",
|
||||
args=["-y", "@modelcontextprotocol/server-brave-search"],
|
||||
env={"BRAVE_API_KEY": "your-key"}
|
||||
)
|
||||
)
|
||||
|
||||
# Multiple MCP servers + regular functions
|
||||
def my_custom_tool(query: str) -> str:
|
||||
"""Custom tool function."""
|
||||
return f"Result: {query}"
|
||||
|
||||
agent = Agent(
|
||||
name="MultiToolAgent",
|
||||
instructions="Agent with multiple MCP servers",
|
||||
tools=[
|
||||
MCP("uvx mcp-server-time"), # Time tools
|
||||
MCP("npx @modelcontextprotocol/server-memory"), # Memory tools
|
||||
my_custom_tool # Regular function
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### MCP Server (Expose Tools as MCP Server)
|
||||
|
||||
Expose your Python functions as MCP tools for Claude Desktop, Cursor, and other MCP clients:
|
||||
|
||||
```python
|
||||
from praisonaiagents.mcp import ToolsMCPServer
|
||||
|
||||
def search_web(query: str, max_results: int = 5) -> dict:
|
||||
"""Search the web for information."""
|
||||
return {"results": [f"Result for {query}"]}
|
||||
|
||||
def calculate(expression: str) -> dict:
|
||||
"""Evaluate a mathematical expression."""
|
||||
return {"result": eval(expression)}
|
||||
|
||||
# Create and run MCP server
|
||||
server = ToolsMCPServer(name="my-tools")
|
||||
server.register_tools([search_web, calculate])
|
||||
server.run() # stdio for Claude Desktop
|
||||
# server.run_sse(host="0.0.0.0", port=8080) # SSE for web clients
|
||||
```
|
||||
|
||||
### MCP Features
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| Session Management | Automatic Mcp-Session-Id handling |
|
||||
| Protocol Versioning | Mcp-Protocol-Version header |
|
||||
| Resumability | SSE stream recovery via Last-Event-ID |
|
||||
| Security | Origin validation, DNS rebinding prevention |
|
||||
| WebSocket | Auto-reconnect with exponential backoff |
|
||||
|
||||
## A2A (Agent2Agent Protocol)
|
||||
|
||||
PraisonAI supports the [A2A Protocol](https://a2a-protocol.org) for agent-to-agent communication, enabling your agents to be discovered and collaborate with other AI agents.
|
||||
|
||||
### A2A Server (Expose Agent as A2A Server)
|
||||
```python
|
||||
from praisonaiagents import Agent, A2A
|
||||
from fastapi import FastAPI
|
||||
|
||||
# Create an agent with tools
|
||||
def search_web(query: str) -> str:
|
||||
"""Search the web for information."""
|
||||
return f"Results for: {query}"
|
||||
|
||||
agent = Agent(
|
||||
name="Research Assistant",
|
||||
role="Research Analyst",
|
||||
goal="Help users research topics",
|
||||
tools=[search_web]
|
||||
)
|
||||
|
||||
# Expose as A2A Server
|
||||
a2a = A2A(agent=agent, url="http://localhost:8000/a2a")
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(a2a.get_router())
|
||||
|
||||
# Run: uvicorn app:app --reload
|
||||
# Agent Card: GET /.well-known/agent.json
|
||||
# Status: GET /status
|
||||
```
|
||||
|
||||
### A2A Features
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| Agent Card | JSON metadata for agent discovery |
|
||||
| Skills Extraction | Auto-generate skills from tools |
|
||||
| Task Management | Stateful task lifecycle |
|
||||
| Streaming | SSE streaming for real-time updates |
|
||||
|
||||
> **Documentation**: [docs.praison.ai/a2a](https://docs.praison.ai/a2a) | **Examples**: [examples/python/a2a](https://github.com/MervinPraison/PraisonAI/tree/main/examples/python/a2a)
|
||||
|
||||
## CLI Features
|
||||
|
||||
| Feature | Docs |
|
||||
@@ -2399,6 +2456,7 @@ PraisonAI provides zero-dependency persistent memory for agents. For detailed ex
|
||||
- 🤝 **Agent Handoffs** - Transfer context between specialised agents
|
||||
- 🛡️ **Guardrails** - Input/output validation and safety checks
|
||||
- ✅ **Human Approval** - Require human confirmation for critical actions
|
||||
- 🔐 **Tool Approval CLI** - `--trust` (auto-approve all) and `--approve-level` (risk-based approval)
|
||||
- 💬 **Sessions Management** - Isolated conversation contexts
|
||||
- 🔄 **Stateful Agents** - Maintain state across interactions
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# PraisonAI CLI - Benchmark Results
|
||||
|
||||
**Generated:** 2025-12-19 16:04:40
|
||||
|
||||
## Import Times
|
||||
|
||||
| Module | Time (ms) | Target | Status |
|
||||
|--------|-----------|--------|--------|
|
||||
| praisonai | 4.11 | <10 | ✅ |
|
||||
| message_queue | 6.87 | <50 | ✅ |
|
||||
| at_mentions | 0.58 | <50 | ✅ |
|
||||
| profiler | 0.90 | <10 | ✅ |
|
||||
|
||||
## Instantiation Times
|
||||
|
||||
| Class | Time (μs) | Target | Status |
|
||||
|-------|-----------|--------|--------|
|
||||
| MessageQueue | 0.26 | <100 | ✅ |
|
||||
| StateManager | 0.30 | <100 | ✅ |
|
||||
| FileSearchService | 8.42 | <100 | ✅ |
|
||||
|
||||
## Lazy Loading
|
||||
|
||||
| Operation | Time (ms) |
|
||||
|-----------|----------|
|
||||
| PraisonAI lazy load | 3886.47 |
|
||||
|
||||
## How to Reproduce
|
||||
|
||||
```bash
|
||||
cd praisonai
|
||||
python benchmarks/simple_benchmark.py
|
||||
```
|
||||
@@ -0,0 +1,231 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
PraisonAI CLI - Performance Benchmark
|
||||
|
||||
Measures import times and module loading performance.
|
||||
|
||||
Usage:
|
||||
python benchmarks/simple_benchmark.py
|
||||
python benchmarks/simple_benchmark.py --no-save
|
||||
"""
|
||||
|
||||
import time
|
||||
import sys
|
||||
import os
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
ITERATIONS = 100
|
||||
|
||||
|
||||
def measure_import(module_name: str) -> float:
|
||||
"""Measure import time in milliseconds."""
|
||||
# Clear from cache if present
|
||||
modules_to_remove = [k for k in sys.modules.keys() if k.startswith(module_name)]
|
||||
for mod in modules_to_remove:
|
||||
del sys.modules[mod]
|
||||
|
||||
start = time.perf_counter()
|
||||
__import__(module_name)
|
||||
return (time.perf_counter() - start) * 1000
|
||||
|
||||
|
||||
def measure_instantiation(create_fn, iterations=ITERATIONS) -> float:
|
||||
"""Measure average instantiation time in microseconds."""
|
||||
times = []
|
||||
for _ in range(iterations):
|
||||
start = time.perf_counter()
|
||||
create_fn()
|
||||
times.append((time.perf_counter() - start) * 1_000_000)
|
||||
return sum(times) / len(times)
|
||||
|
||||
|
||||
def run_benchmark():
|
||||
"""Run the benchmark."""
|
||||
results = {}
|
||||
|
||||
print('=' * 60)
|
||||
print('PraisonAI CLI - Performance Benchmark')
|
||||
print('=' * 60)
|
||||
print(f'\nIterations: {ITERATIONS}')
|
||||
print('Metrics: Import time (ms), Instantiation time (μs)\n')
|
||||
|
||||
# 1. Package import time
|
||||
print("1. Testing package imports...")
|
||||
|
||||
# Clear cache
|
||||
modules_to_remove = [k for k in sys.modules.keys() if 'praisonai' in k]
|
||||
for mod in modules_to_remove:
|
||||
del sys.modules[mod]
|
||||
|
||||
start = time.perf_counter()
|
||||
import praisonai
|
||||
results['praisonai_import_ms'] = (time.perf_counter() - start) * 1000
|
||||
print(f" import praisonai: {results['praisonai_import_ms']:.2f}ms")
|
||||
|
||||
# 2. CLI features import
|
||||
print("\n2. Testing CLI features imports...")
|
||||
|
||||
start = time.perf_counter()
|
||||
from praisonai.cli.features.message_queue import MessageQueue
|
||||
results['message_queue_import_ms'] = (time.perf_counter() - start) * 1000
|
||||
print(f" message_queue: {results['message_queue_import_ms']:.2f}ms")
|
||||
|
||||
start = time.perf_counter()
|
||||
from praisonai.cli.features.at_mentions import CombinedCompleter
|
||||
results['at_mentions_import_ms'] = (time.perf_counter() - start) * 1000
|
||||
print(f" at_mentions: {results['at_mentions_import_ms']:.2f}ms")
|
||||
|
||||
# 3. Profiler import (should be fast, standalone)
|
||||
print("\n3. Testing profiler import...")
|
||||
start = time.perf_counter()
|
||||
from praisonai.profiler import Profiler
|
||||
results['profiler_import_ms'] = (time.perf_counter() - start) * 1000
|
||||
print(f" profiler: {results['profiler_import_ms']:.2f}ms")
|
||||
|
||||
# 4. Instantiation benchmarks
|
||||
print(f"\n4. Testing instantiation ({ITERATIONS} iterations)...")
|
||||
|
||||
from praisonai.cli.features.message_queue import MessageQueue, StateManager
|
||||
from praisonai.cli.features.at_mentions import FileSearchService
|
||||
|
||||
results['MessageQueue_us'] = measure_instantiation(lambda: MessageQueue())
|
||||
print(f" MessageQueue: {results['MessageQueue_us']:.2f}μs")
|
||||
|
||||
results['StateManager_us'] = measure_instantiation(lambda: StateManager())
|
||||
print(f" StateManager: {results['StateManager_us']:.2f}μs")
|
||||
|
||||
results['FileSearchService_us'] = measure_instantiation(
|
||||
lambda: FileSearchService(root_dir=".")
|
||||
)
|
||||
print(f" FileSearchService: {results['FileSearchService_us']:.2f}μs")
|
||||
|
||||
# 5. PraisonAI class (lazy load)
|
||||
print("\n5. Testing PraisonAI class access (lazy load)...")
|
||||
start = time.perf_counter()
|
||||
PraisonAI = praisonai.PraisonAI
|
||||
results['PraisonAI_lazy_ms'] = (time.perf_counter() - start) * 1000
|
||||
print(f" PraisonAI lazy load: {results['PraisonAI_lazy_ms']:.2f}ms")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def print_summary(results: dict):
|
||||
"""Print benchmark summary."""
|
||||
print('\n' + '=' * 60)
|
||||
print('SUMMARY')
|
||||
print('=' * 60)
|
||||
|
||||
print('\nImport Times (target: <50ms for features, <10ms for package):')
|
||||
print(f" {'Module':<25} {'Time':>10} {'Status':>10}")
|
||||
print(f" {'-'*25} {'-'*10} {'-'*10}")
|
||||
|
||||
import_checks = [
|
||||
('praisonai', results['praisonai_import_ms'], 10),
|
||||
('message_queue', results['message_queue_import_ms'], 50),
|
||||
('at_mentions', results['at_mentions_import_ms'], 50),
|
||||
('profiler', results['profiler_import_ms'], 10),
|
||||
]
|
||||
|
||||
all_pass = True
|
||||
for name, time_ms, threshold in import_checks:
|
||||
status = '✅ PASS' if time_ms < threshold else '❌ SLOW'
|
||||
if time_ms >= threshold:
|
||||
all_pass = False
|
||||
print(f" {name:<25} {time_ms:>8.2f}ms {status:>10}")
|
||||
|
||||
print('\nInstantiation Times (target: <100μs):')
|
||||
print(f" {'Class':<25} {'Time':>10} {'Status':>10}")
|
||||
print(f" {'-'*25} {'-'*10} {'-'*10}")
|
||||
|
||||
inst_checks = [
|
||||
('MessageQueue', results['MessageQueue_us'], 100),
|
||||
('StateManager', results['StateManager_us'], 100),
|
||||
('FileSearchService', results['FileSearchService_us'], 100),
|
||||
]
|
||||
|
||||
for name, time_us, threshold in inst_checks:
|
||||
status = '✅ PASS' if time_us < threshold else '❌ SLOW'
|
||||
if time_us >= threshold:
|
||||
all_pass = False
|
||||
print(f" {name:<25} {time_us:>8.2f}μs {status:>10}")
|
||||
|
||||
print('\n' + '=' * 60)
|
||||
if all_pass:
|
||||
print('✅ All benchmarks PASSED')
|
||||
else:
|
||||
print('❌ Some benchmarks FAILED')
|
||||
print('=' * 60)
|
||||
|
||||
return all_pass
|
||||
|
||||
|
||||
def save_results(results: dict):
|
||||
"""Save benchmark results to markdown file."""
|
||||
filepath = os.path.join(os.path.dirname(__file__), 'BENCHMARK_RESULTS.md')
|
||||
|
||||
with open(filepath, 'w') as f:
|
||||
f.write('# PraisonAI CLI - Benchmark Results\n\n')
|
||||
f.write(f'**Generated:** {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}\n\n')
|
||||
|
||||
f.write('## Import Times\n\n')
|
||||
f.write('| Module | Time (ms) | Target | Status |\n')
|
||||
f.write('|--------|-----------|--------|--------|\n')
|
||||
|
||||
import_data = [
|
||||
('praisonai', results['praisonai_import_ms'], 10),
|
||||
('message_queue', results['message_queue_import_ms'], 50),
|
||||
('at_mentions', results['at_mentions_import_ms'], 50),
|
||||
('profiler', results['profiler_import_ms'], 10),
|
||||
]
|
||||
|
||||
for name, time_ms, threshold in import_data:
|
||||
status = '✅' if time_ms < threshold else '❌'
|
||||
f.write(f'| {name} | {time_ms:.2f} | <{threshold} | {status} |\n')
|
||||
|
||||
f.write('\n## Instantiation Times\n\n')
|
||||
f.write('| Class | Time (μs) | Target | Status |\n')
|
||||
f.write('|-------|-----------|--------|--------|\n')
|
||||
|
||||
inst_data = [
|
||||
('MessageQueue', results['MessageQueue_us'], 100),
|
||||
('StateManager', results['StateManager_us'], 100),
|
||||
('FileSearchService', results['FileSearchService_us'], 100),
|
||||
]
|
||||
|
||||
for name, time_us, threshold in inst_data:
|
||||
status = '✅' if time_us < threshold else '❌'
|
||||
f.write(f'| {name} | {time_us:.2f} | <{threshold} | {status} |\n')
|
||||
|
||||
f.write('\n## Lazy Loading\n\n')
|
||||
f.write(f'| Operation | Time (ms) |\n')
|
||||
f.write(f'|-----------|----------|\n')
|
||||
f.write(f'| PraisonAI lazy load | {results["PraisonAI_lazy_ms"]:.2f} |\n')
|
||||
|
||||
f.write('\n## How to Reproduce\n\n')
|
||||
f.write('```bash\n')
|
||||
f.write('cd praisonai\n')
|
||||
f.write('python benchmarks/simple_benchmark.py\n')
|
||||
f.write('```\n')
|
||||
|
||||
print(f'\nResults saved to: {filepath}')
|
||||
return filepath
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='PraisonAI CLI Performance Benchmark')
|
||||
parser.add_argument('--no-save', action='store_true', help='Do not save results to file')
|
||||
args = parser.parse_args()
|
||||
|
||||
results = run_benchmark()
|
||||
all_pass = print_summary(results)
|
||||
|
||||
if not args.no_save:
|
||||
save_results(results)
|
||||
|
||||
return 0 if all_pass else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -3,8 +3,8 @@ class Praisonai < Formula
|
||||
|
||||
desc "AI tools for various AI applications"
|
||||
homepage "https://github.com/MervinPraison/PraisonAI"
|
||||
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.52.tar.gz"
|
||||
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.52.tar.gz | shasum -a 256`.split.first
|
||||
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.85.tar.gz"
|
||||
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.85.tar.gz | shasum -a 256`.split.first
|
||||
license "MIT"
|
||||
|
||||
depends_on "[email protected]"
|
||||
|
||||
@@ -6,30 +6,29 @@ logging.getLogger('crewai.cli.config').setLevel(logging.ERROR)
|
||||
import os
|
||||
os.environ["OTEL_SDK_DISABLED"] = "true"
|
||||
os.environ["EC_TELEMETRY"] = "false"
|
||||
from .cli import PraisonAI
|
||||
|
||||
# Version is lightweight, import directly
|
||||
from .version import __version__
|
||||
|
||||
# Re-export all classes from praisonaiagents to enable:
|
||||
# from praisonai import Agent, Task, PraisonAIAgents
|
||||
try:
|
||||
import praisonaiagents
|
||||
# Import all symbols from praisonaiagents using * import
|
||||
from praisonaiagents import *
|
||||
except ImportError:
|
||||
# If praisonaiagents is not available, these imports will fail gracefully
|
||||
pass
|
||||
|
||||
# Define __all__ to include both PraisonAI core classes and praisonaiagents exports
|
||||
# Define __all__ for lazy loading
|
||||
__all__ = [
|
||||
# Core PraisonAI classes
|
||||
'PraisonAI',
|
||||
'__version__',
|
||||
]
|
||||
|
||||
# Dynamically extend __all__ with praisonaiagents exports
|
||||
try:
|
||||
import praisonaiagents
|
||||
__all__.extend(praisonaiagents.__all__)
|
||||
except (ImportError, AttributeError):
|
||||
# If praisonaiagents is not available or doesn't have __all__, fail gracefully
|
||||
pass
|
||||
# Lazy loading for heavy imports
|
||||
def __getattr__(name):
|
||||
"""Lazy load heavy modules to improve import time."""
|
||||
if name == 'PraisonAI':
|
||||
from .cli import PraisonAI
|
||||
return PraisonAI
|
||||
|
||||
# Try praisonaiagents exports
|
||||
try:
|
||||
import praisonaiagents
|
||||
if hasattr(praisonaiagents, name):
|
||||
return getattr(praisonaiagents, name)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -0,0 +1,337 @@
|
||||
"""
|
||||
Agent Scheduler for PraisonAI - Run agents periodically 24/7.
|
||||
|
||||
This module provides scheduling capabilities for running PraisonAI agents
|
||||
at regular intervals, enabling 24/7 autonomous agent operations.
|
||||
|
||||
Example:
|
||||
# Run news checker every hour
|
||||
from praisonai.agent_scheduler import AgentScheduler
|
||||
from praisonai_agents import Agent
|
||||
|
||||
agent = Agent(
|
||||
name="NewsChecker",
|
||||
instructions="Check latest AI news and summarize",
|
||||
tools=[search_tool]
|
||||
)
|
||||
|
||||
scheduler = AgentScheduler(agent, task="Check latest AI news")
|
||||
scheduler.start(schedule_expr="hourly")
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Optional, Dict, Any, Callable
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ScheduleParser:
|
||||
"""Parse schedule expressions into intervals."""
|
||||
|
||||
@staticmethod
|
||||
def parse(schedule_expr: str) -> int:
|
||||
"""
|
||||
Parse schedule expression and return interval in seconds.
|
||||
|
||||
Supported formats:
|
||||
- "daily" -> 86400 seconds
|
||||
- "hourly" -> 3600 seconds
|
||||
- "*/30m" -> 1800 seconds (every 30 minutes)
|
||||
- "*/1h" -> 3600 seconds (every 1 hour)
|
||||
- "60" -> 60 seconds (plain number)
|
||||
|
||||
Args:
|
||||
schedule_expr: Schedule expression string
|
||||
|
||||
Returns:
|
||||
Interval in seconds
|
||||
|
||||
Raises:
|
||||
ValueError: If schedule format is not supported
|
||||
"""
|
||||
schedule_expr = schedule_expr.strip().lower()
|
||||
|
||||
if schedule_expr == "daily":
|
||||
return 86400
|
||||
elif schedule_expr == "hourly":
|
||||
return 3600
|
||||
elif schedule_expr.isdigit():
|
||||
return int(schedule_expr)
|
||||
elif schedule_expr.startswith("*/"):
|
||||
interval_part = schedule_expr[2:]
|
||||
if interval_part.endswith("m"):
|
||||
minutes = int(interval_part[:-1])
|
||||
return minutes * 60
|
||||
elif interval_part.endswith("h"):
|
||||
hours = int(interval_part[:-1])
|
||||
return hours * 3600
|
||||
elif interval_part.endswith("s"):
|
||||
return int(interval_part[:-1])
|
||||
else:
|
||||
return int(interval_part)
|
||||
else:
|
||||
raise ValueError(f"Unsupported schedule format: {schedule_expr}")
|
||||
|
||||
|
||||
class AgentExecutorInterface(ABC):
|
||||
"""Abstract interface for agent execution."""
|
||||
|
||||
@abstractmethod
|
||||
def execute(self, task: str) -> Any:
|
||||
"""Execute the agent with given task."""
|
||||
pass
|
||||
|
||||
|
||||
class PraisonAgentExecutor(AgentExecutorInterface):
|
||||
"""Executor for PraisonAI agents."""
|
||||
|
||||
def __init__(self, agent):
|
||||
"""
|
||||
Initialize executor with a PraisonAI agent.
|
||||
|
||||
Args:
|
||||
agent: PraisonAI Agent instance
|
||||
"""
|
||||
self.agent = agent
|
||||
|
||||
def execute(self, task: str) -> Any:
|
||||
"""
|
||||
Execute the agent with given task.
|
||||
|
||||
Args:
|
||||
task: Task description for the agent
|
||||
|
||||
Returns:
|
||||
Agent execution result
|
||||
"""
|
||||
try:
|
||||
result = self.agent.start(task)
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"Agent execution failed: {e}")
|
||||
raise
|
||||
|
||||
|
||||
class AgentScheduler:
|
||||
"""
|
||||
Scheduler for running PraisonAI agents periodically.
|
||||
|
||||
Features:
|
||||
- Interval-based scheduling (hourly, daily, custom)
|
||||
- Thread-safe operation
|
||||
- Automatic retry on failure
|
||||
- Execution logging and monitoring
|
||||
- Graceful shutdown
|
||||
|
||||
Example:
|
||||
scheduler = AgentScheduler(agent, task="Check news")
|
||||
scheduler.start(schedule_expr="hourly", max_retries=3)
|
||||
# Agent runs every hour automatically
|
||||
scheduler.stop() # Stop when needed
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
agent,
|
||||
task: str,
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
on_success: Optional[Callable] = None,
|
||||
on_failure: Optional[Callable] = None
|
||||
):
|
||||
"""
|
||||
Initialize agent scheduler.
|
||||
|
||||
Args:
|
||||
agent: PraisonAI Agent instance
|
||||
task: Task description to execute
|
||||
config: Optional configuration dict
|
||||
on_success: Callback function on successful execution
|
||||
on_failure: Callback function on failed execution
|
||||
"""
|
||||
self.agent = agent
|
||||
self.task = task
|
||||
self.config = config or {}
|
||||
self.on_success = on_success
|
||||
self.on_failure = on_failure
|
||||
|
||||
self.is_running = False
|
||||
self._stop_event = threading.Event()
|
||||
self._thread = None
|
||||
self._executor = PraisonAgentExecutor(agent)
|
||||
self._execution_count = 0
|
||||
self._success_count = 0
|
||||
self._failure_count = 0
|
||||
|
||||
def start(
|
||||
self,
|
||||
schedule_expr: str,
|
||||
max_retries: int = 3,
|
||||
run_immediately: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
Start scheduled agent execution.
|
||||
|
||||
Args:
|
||||
schedule_expr: Schedule expression (e.g., "hourly", "*/1h", "3600")
|
||||
max_retries: Maximum retry attempts on failure
|
||||
run_immediately: If True, run agent immediately before starting schedule
|
||||
|
||||
Returns:
|
||||
True if scheduler started successfully
|
||||
"""
|
||||
if self.is_running:
|
||||
logger.warning("Scheduler is already running")
|
||||
return False
|
||||
|
||||
try:
|
||||
interval = ScheduleParser.parse(schedule_expr)
|
||||
self.is_running = True
|
||||
self._stop_event.clear()
|
||||
|
||||
logger.info(f"Starting agent scheduler: {self.agent.name if hasattr(self.agent, 'name') else 'Agent'}")
|
||||
logger.info(f"Task: {self.task}")
|
||||
logger.info(f"Schedule: {schedule_expr} ({interval}s interval)")
|
||||
logger.info(f"Max retries: {max_retries}")
|
||||
|
||||
# Run immediately if requested
|
||||
if run_immediately:
|
||||
logger.info("Running agent immediately before starting schedule...")
|
||||
self._execute_with_retry(max_retries)
|
||||
|
||||
self._thread = threading.Thread(
|
||||
target=self._run_schedule,
|
||||
args=(interval, max_retries),
|
||||
daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
logger.info("Agent scheduler started successfully")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start scheduler: {e}")
|
||||
self.is_running = False
|
||||
return False
|
||||
|
||||
def stop(self) -> bool:
|
||||
"""
|
||||
Stop the scheduler gracefully.
|
||||
|
||||
Returns:
|
||||
True if stopped successfully
|
||||
"""
|
||||
if not self.is_running:
|
||||
logger.info("Scheduler is not running")
|
||||
return True
|
||||
|
||||
logger.info("Stopping agent scheduler...")
|
||||
self._stop_event.set()
|
||||
|
||||
if self._thread and self._thread.is_alive():
|
||||
self._thread.join(timeout=10)
|
||||
|
||||
self.is_running = False
|
||||
logger.info("Agent scheduler stopped")
|
||||
logger.info(f"Execution stats - Total: {self._execution_count}, Success: {self._success_count}, Failed: {self._failure_count}")
|
||||
return True
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get execution statistics.
|
||||
|
||||
Returns:
|
||||
Dictionary with execution stats
|
||||
"""
|
||||
return {
|
||||
"is_running": self.is_running,
|
||||
"total_executions": self._execution_count,
|
||||
"successful_executions": self._success_count,
|
||||
"failed_executions": self._failure_count,
|
||||
"success_rate": (self._success_count / self._execution_count * 100) if self._execution_count > 0 else 0
|
||||
}
|
||||
|
||||
def _run_schedule(self, interval: int, max_retries: int):
|
||||
"""Internal method to run scheduled agent executions."""
|
||||
while not self._stop_event.is_set():
|
||||
logger.info(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Starting scheduled agent execution")
|
||||
|
||||
self._execute_with_retry(max_retries)
|
||||
|
||||
# Wait for next scheduled time
|
||||
logger.info(f"Next execution in {interval} seconds ({interval/3600:.1f} hours)")
|
||||
self._stop_event.wait(interval)
|
||||
|
||||
def _execute_with_retry(self, max_retries: int):
|
||||
"""Execute agent with retry logic."""
|
||||
self._execution_count += 1
|
||||
success = False
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
logger.info(f"Attempt {attempt + 1}/{max_retries}")
|
||||
result = self._executor.execute(self.task)
|
||||
|
||||
logger.info(f"Agent execution successful on attempt {attempt + 1}")
|
||||
logger.info(f"Result: {result}")
|
||||
|
||||
self._success_count += 1
|
||||
success = True
|
||||
|
||||
if self.on_success:
|
||||
self.on_success(result)
|
||||
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Agent execution failed on attempt {attempt + 1}: {e}")
|
||||
|
||||
if attempt < max_retries - 1:
|
||||
wait_time = 30 * (attempt + 1) # Exponential backoff
|
||||
logger.info(f"Waiting {wait_time}s before retry...")
|
||||
time.sleep(wait_time)
|
||||
|
||||
if not success:
|
||||
self._failure_count += 1
|
||||
logger.error(f"Agent execution failed after {max_retries} attempts")
|
||||
|
||||
if self.on_failure:
|
||||
self.on_failure(f"Failed after {max_retries} attempts")
|
||||
|
||||
def execute_once(self) -> Any:
|
||||
"""
|
||||
Execute agent immediately (one-time execution).
|
||||
|
||||
Returns:
|
||||
Agent execution result
|
||||
"""
|
||||
logger.info("Executing agent once")
|
||||
try:
|
||||
result = self._executor.execute(self.task)
|
||||
logger.info(f"One-time execution successful: {result}")
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"One-time execution failed: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def create_agent_scheduler(
|
||||
agent,
|
||||
task: str,
|
||||
config: Optional[Dict[str, Any]] = None
|
||||
) -> AgentScheduler:
|
||||
"""
|
||||
Factory function to create agent scheduler.
|
||||
|
||||
Args:
|
||||
agent: PraisonAI Agent instance
|
||||
task: Task description
|
||||
config: Optional configuration
|
||||
|
||||
Returns:
|
||||
Configured AgentScheduler instance
|
||||
"""
|
||||
return AgentScheduler(agent, task, config)
|
||||
+255
-18
@@ -62,7 +62,7 @@ except ImportError:
|
||||
logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO').upper(), format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
|
||||
# =============================================================================
|
||||
# Available Tools List (shared between generators)
|
||||
# Available Tools List (shared between generators) - Legacy for praisonai_tools
|
||||
# =============================================================================
|
||||
AVAILABLE_TOOLS = [
|
||||
"CodeDocsSearchTool", "CSVSearchTool", "DirectorySearchTool", "DOCXSearchTool",
|
||||
@@ -72,6 +72,206 @@ AVAILABLE_TOOLS = [
|
||||
"YoutubeChannelSearchTool", "YoutubeVideoSearchTool"
|
||||
]
|
||||
|
||||
# =============================================================================
|
||||
# Enhanced Tool Discovery from praisonaiagents.tools
|
||||
# =============================================================================
|
||||
|
||||
# Tool categories with their tools from praisonaiagents.tools
|
||||
TOOL_CATEGORIES = {
|
||||
'web_search': [
|
||||
'internet_search', 'duckduckgo', 'tavily_search', 'exa_search',
|
||||
'search_web', 'ydc_search', 'searxng_search'
|
||||
],
|
||||
'web_scraping': [
|
||||
'scrape_page', 'extract_links', 'crawl', 'extract_text',
|
||||
'crawl4ai', 'crawl4ai_extract', 'get_article'
|
||||
],
|
||||
'file_operations': [
|
||||
'read_file', 'write_file', 'list_files', 'get_file_info',
|
||||
'copy_file', 'move_file', 'delete_file'
|
||||
],
|
||||
'code_execution': [
|
||||
'execute_command', 'execute_code', 'analyze_code', 'format_code'
|
||||
],
|
||||
'data_processing': [
|
||||
'read_csv', 'write_csv', 'analyze_csv', 'read_json', 'write_json',
|
||||
'read_excel', 'write_excel', 'read_yaml', 'write_yaml', 'read_xml'
|
||||
],
|
||||
'research': [
|
||||
'search_arxiv', 'get_arxiv_paper', 'wiki_search', 'wiki_summary',
|
||||
'get_news_sources', 'get_trending_topics'
|
||||
],
|
||||
'finance': [
|
||||
'get_stock_price', 'get_stock_info', 'get_historical_data'
|
||||
],
|
||||
'math': [
|
||||
'evaluate', 'solve_equation', 'convert_units', 'calculate_statistics'
|
||||
],
|
||||
'database': [
|
||||
'query', 'create_table', 'load_data', 'find_documents', 'vector_search'
|
||||
]
|
||||
}
|
||||
|
||||
# Keywords that map to tool categories
|
||||
TASK_KEYWORD_TO_TOOLS = {
|
||||
# Web search keywords
|
||||
'search': 'web_search',
|
||||
'find': 'web_search',
|
||||
'look up': 'web_search',
|
||||
'google': 'web_search',
|
||||
'internet': 'web_search',
|
||||
'online': 'web_search',
|
||||
'web': 'web_search',
|
||||
|
||||
# Web scraping keywords
|
||||
'scrape': 'web_scraping',
|
||||
'crawl': 'web_scraping',
|
||||
'extract from website': 'web_scraping',
|
||||
'get from url': 'web_scraping',
|
||||
'fetch page': 'web_scraping',
|
||||
|
||||
# File operation keywords
|
||||
'read file': 'file_operations',
|
||||
'write file': 'file_operations',
|
||||
'save': 'file_operations',
|
||||
'load': 'file_operations',
|
||||
'open file': 'file_operations',
|
||||
'create file': 'file_operations',
|
||||
|
||||
# Code execution keywords
|
||||
'execute': 'code_execution',
|
||||
'run code': 'code_execution',
|
||||
'python': 'code_execution',
|
||||
'script': 'code_execution',
|
||||
'command': 'code_execution',
|
||||
'shell': 'code_execution',
|
||||
|
||||
# Data processing keywords
|
||||
'csv': 'data_processing',
|
||||
'excel': 'data_processing',
|
||||
'json': 'data_processing',
|
||||
'yaml': 'data_processing',
|
||||
'xml': 'data_processing',
|
||||
'data': 'data_processing',
|
||||
'spreadsheet': 'data_processing',
|
||||
|
||||
# Research keywords
|
||||
'research': 'research',
|
||||
'paper': 'research',
|
||||
'arxiv': 'research',
|
||||
'wikipedia': 'research',
|
||||
'academic': 'research',
|
||||
'news': 'research',
|
||||
|
||||
# Finance keywords
|
||||
'stock': 'finance',
|
||||
'price': 'finance',
|
||||
'market': 'finance',
|
||||
'financial': 'finance',
|
||||
'trading': 'finance',
|
||||
|
||||
# Math keywords
|
||||
'calculate': 'math',
|
||||
'math': 'math',
|
||||
'equation': 'math',
|
||||
'compute': 'math',
|
||||
'statistics': 'math',
|
||||
|
||||
# Database keywords
|
||||
'database': 'database',
|
||||
'sql': 'database',
|
||||
'query': 'database',
|
||||
'mongodb': 'database',
|
||||
'vector': 'database'
|
||||
}
|
||||
|
||||
|
||||
def get_all_available_tools() -> Dict[str, List[str]]:
|
||||
"""
|
||||
Get all available tools organized by category.
|
||||
|
||||
Returns:
|
||||
Dict mapping category names to lists of tool names
|
||||
"""
|
||||
return TOOL_CATEGORIES.copy()
|
||||
|
||||
|
||||
def get_tools_for_task(task_description: str) -> List[str]:
|
||||
"""
|
||||
Analyze a task description and return appropriate tools.
|
||||
|
||||
Args:
|
||||
task_description: The task to analyze
|
||||
|
||||
Returns:
|
||||
List of tool names appropriate for the task
|
||||
"""
|
||||
task_lower = task_description.lower()
|
||||
matched_categories = set()
|
||||
|
||||
# Match keywords to categories
|
||||
for keyword, category in TASK_KEYWORD_TO_TOOLS.items():
|
||||
if keyword in task_lower:
|
||||
matched_categories.add(category)
|
||||
|
||||
# Collect tools from matched categories
|
||||
tools = []
|
||||
for category in matched_categories:
|
||||
if category in TOOL_CATEGORIES:
|
||||
tools.extend(TOOL_CATEGORIES[category])
|
||||
|
||||
# Always include core tools for flexibility
|
||||
core_tools = ['read_file', 'write_file', 'execute_command']
|
||||
for tool in core_tools:
|
||||
if tool not in tools:
|
||||
tools.append(tool)
|
||||
|
||||
# Remove duplicates while preserving order
|
||||
seen = set()
|
||||
unique_tools = []
|
||||
for tool in tools:
|
||||
if tool not in seen:
|
||||
seen.add(tool)
|
||||
unique_tools.append(tool)
|
||||
|
||||
return unique_tools
|
||||
|
||||
|
||||
def recommend_agent_count(task_description: str) -> int:
|
||||
"""
|
||||
Recommend the optimal number of agents based on task complexity.
|
||||
|
||||
Args:
|
||||
task_description: The task to analyze
|
||||
|
||||
Returns:
|
||||
Recommended number of agents (1-4)
|
||||
"""
|
||||
complexity = BaseAutoGenerator.analyze_complexity(task_description)
|
||||
|
||||
if complexity == 'simple':
|
||||
return 1
|
||||
elif complexity == 'moderate':
|
||||
return 2
|
||||
else: # complex
|
||||
# Count distinct aspects of the task
|
||||
task_lower = task_description.lower()
|
||||
aspects = 0
|
||||
|
||||
aspect_keywords = [
|
||||
['research', 'search', 'find', 'gather'],
|
||||
['analyze', 'evaluate', 'assess', 'review'],
|
||||
['write', 'create', 'generate', 'produce'],
|
||||
['edit', 'refine', 'improve', 'polish'],
|
||||
['coordinate', 'manage', 'orchestrate', 'delegate']
|
||||
]
|
||||
|
||||
for keyword_group in aspect_keywords:
|
||||
if any(kw in task_lower for kw in keyword_group):
|
||||
aspects += 1
|
||||
|
||||
return min(max(aspects, 2), 4) # Between 2 and 4 agents
|
||||
|
||||
# =============================================================================
|
||||
# Base Generator Class (DRY - shared functionality)
|
||||
# =============================================================================
|
||||
@@ -458,9 +658,19 @@ Tools are not available for {framework}. To use tools, install:
|
||||
|
||||
return merged_data
|
||||
|
||||
def discover_tools_for_topic(self) -> List[str]:
|
||||
"""
|
||||
Discover appropriate tools for the topic using intelligent matching.
|
||||
|
||||
Returns:
|
||||
List of tool names appropriate for this topic
|
||||
"""
|
||||
return get_tools_for_task(self.topic)
|
||||
|
||||
def get_user_content(self):
|
||||
"""
|
||||
Generates a prompt for the OpenAI API to generate a team structure.
|
||||
Uses intelligent tool discovery based on task analysis.
|
||||
|
||||
Args:
|
||||
None
|
||||
@@ -484,42 +694,68 @@ Tools are not available for {framework}. To use tools, install:
|
||||
|
||||
workflow_guidance = pattern_guidance.get(self.pattern, pattern_guidance["sequential"])
|
||||
|
||||
# Get recommended tools based on task analysis
|
||||
recommended_tools = self.discover_tools_for_topic()
|
||||
recommended_agent_count = recommend_agent_count(self.topic)
|
||||
complexity = self.analyze_complexity(self.topic)
|
||||
|
||||
# Build comprehensive tool list with categories
|
||||
all_tools_by_category = []
|
||||
for category, tools in TOOL_CATEGORIES.items():
|
||||
all_tools_by_category.append(f" {category}: {', '.join(tools)}")
|
||||
tools_reference = "\n".join(all_tools_by_category)
|
||||
|
||||
# Also include legacy tools for backward compatibility
|
||||
legacy_tools = ", ".join(AVAILABLE_TOOLS)
|
||||
|
||||
user_content = f"""Analyze and generate a team structure for: "{self.topic}"
|
||||
|
||||
STEP 1: TASK COMPLEXITY ANALYSIS
|
||||
First, analyze the task complexity:
|
||||
- Is this a simple task that 1-2 agents could handle?
|
||||
- Does it require multiple specialized perspectives?
|
||||
- Are there clear sequential dependencies between steps?
|
||||
TASK COMPLEXITY ANALYSIS (Pre-computed):
|
||||
- Complexity: {complexity}
|
||||
- Recommended agents: {recommended_agent_count}
|
||||
- Recommended tools based on task keywords: {', '.join(recommended_tools)}
|
||||
|
||||
STEP 1: VALIDATE TASK ANALYSIS
|
||||
Review the pre-computed analysis above. Adjust if needed based on your understanding.
|
||||
|
||||
STEP 2: DETERMINE OPTIMAL TEAM SIZE
|
||||
Based on your analysis, create the minimum number of agents needed:
|
||||
- Simple tasks: 1-2 agents
|
||||
- Moderate tasks: 2-3 agents
|
||||
- Complex tasks: 3-4 agents (maximum)
|
||||
Based on complexity analysis:
|
||||
- Simple tasks: 1-2 agents (single focused agent or simple pair)
|
||||
- Moderate tasks: 2-3 agents (researcher + executor pattern)
|
||||
- Complex tasks: 3-4 agents (specialized team)
|
||||
|
||||
IMPORTANT: Only create agents that provide meaningful specialization. Avoid unnecessary complexity.
|
||||
Recommended for this task: {recommended_agent_count} agent(s)
|
||||
|
||||
IMPORTANT: Avoid unnecessary complexity. Only add agents if there is meaningful specialization.
|
||||
Each agent must have a distinct, non-overlapping responsibility.
|
||||
|
||||
STEP 3: DESIGN THE TEAM (Pattern: {self.pattern})
|
||||
{workflow_guidance}
|
||||
|
||||
Each agent should have:
|
||||
- A clear, distinct role
|
||||
- A clear, distinct role with meaningful specialization
|
||||
- A specific goal
|
||||
- Relevant backstory
|
||||
- 1 focused task with clear description and expected output
|
||||
- Appropriate tools (only if needed)
|
||||
- Appropriate tools from the recommended list
|
||||
|
||||
Available Tools: CodeDocsSearchTool, CSVSearchTool, DirectorySearchTool, DOCXSearchTool, DirectoryReadTool, FileReadTool, TXTSearchTool, JSONSearchTool, MDXSearchTool, PDFSearchTool, RagTool, ScrapeElementFromWebsiteTool, ScrapeWebsiteTool, WebsiteSearchTool, XMLSearchTool, YoutubeChannelSearchTool, YoutubeVideoSearchTool.
|
||||
Only use these tools if the task requires them. Use empty list [] if no tools needed.
|
||||
AVAILABLE TOOLS BY CATEGORY:
|
||||
{tools_reference}
|
||||
|
||||
Example structure (2 agents for a writing task):
|
||||
LEGACY TOOLS (for backward compatibility):
|
||||
{legacy_tools}
|
||||
|
||||
RECOMMENDED TOOLS FOR THIS TASK: {', '.join(recommended_tools)}
|
||||
Prioritize using the recommended tools. Only add others if specifically needed.
|
||||
|
||||
Example structure (2 agents for a research + writing task):
|
||||
{{
|
||||
"roles": {{
|
||||
"researcher": {{
|
||||
"role": "Research Analyst",
|
||||
"goal": "Gather comprehensive information on the topic",
|
||||
"backstory": "Expert researcher skilled at finding and synthesizing information.",
|
||||
"tools": ["WebsiteSearchTool"],
|
||||
"tools": ["internet_search", "read_file"],
|
||||
"tasks": {{
|
||||
"research_task": {{
|
||||
"description": "Research key information about the topic and compile findings.",
|
||||
@@ -531,7 +767,7 @@ Example structure (2 agents for a writing task):
|
||||
"role": "Content Writer",
|
||||
"goal": "Create polished final content",
|
||||
"backstory": "Skilled writer who transforms research into engaging content.",
|
||||
"tools": [],
|
||||
"tools": ["write_file"],
|
||||
"tasks": {{
|
||||
"writing_task": {{
|
||||
"description": "Write the final content based on research findings.",
|
||||
@@ -543,6 +779,7 @@ Example structure (2 agents for a writing task):
|
||||
}}
|
||||
|
||||
Now generate the optimal team structure for: {self.topic}
|
||||
Use the recommended tools: {', '.join(recommended_tools)}
|
||||
"""
|
||||
return user_content
|
||||
|
||||
|
||||
@@ -8,12 +8,19 @@ Structure:
|
||||
- features/: Feature handlers for CLI flags and commands
|
||||
"""
|
||||
|
||||
from .main import PraisonAI
|
||||
|
||||
__all__ = ["PraisonAI"]
|
||||
|
||||
|
||||
def __getattr__(name):
|
||||
"""Lazy load PraisonAI to avoid slow imports."""
|
||||
if name == "PraisonAI":
|
||||
from .main import PraisonAI
|
||||
return PraisonAI
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
def main():
|
||||
"""CLI entry point function."""
|
||||
from .main import PraisonAI
|
||||
praison_ai = PraisonAI()
|
||||
praison_ai.main()
|
||||
@@ -26,6 +26,9 @@ Features:
|
||||
- autonomy_mode: Autonomy levels (suggest, auto-edit, full-auto)
|
||||
- cost_tracker: Real-time cost and token tracking
|
||||
- message_queue: Message queuing while agent is processing
|
||||
- at_mentions: @ mention autocomplete for files and directories
|
||||
- compare: CLI command comparison feature
|
||||
- eval: Agent evaluation framework (accuracy, performance, reliability, criteria)
|
||||
"""
|
||||
|
||||
# Type hints available for IDE support
|
||||
@@ -58,6 +61,16 @@ __all__ = [
|
||||
'GitIntegrationHandler',
|
||||
'SandboxExecutorHandler',
|
||||
'MessageQueueHandler',
|
||||
# @ mention autocomplete
|
||||
'AtMentionCompleter',
|
||||
'CombinedCompleter',
|
||||
'FileSearchService',
|
||||
# Evaluation
|
||||
'EvalHandler',
|
||||
# Compare feature
|
||||
'CompareHandler',
|
||||
# Agent Skills
|
||||
'SkillsHandler',
|
||||
]
|
||||
|
||||
def __getattr__(name):
|
||||
@@ -138,4 +151,23 @@ def __getattr__(name):
|
||||
elif name == 'MessageQueueHandler':
|
||||
from .message_queue import MessageQueueHandler
|
||||
return MessageQueueHandler
|
||||
# @ mention autocomplete
|
||||
elif name == 'AtMentionCompleter':
|
||||
from .at_mentions import AtMentionCompleter
|
||||
return AtMentionCompleter
|
||||
elif name == 'CombinedCompleter':
|
||||
from .at_mentions import CombinedCompleter
|
||||
return CombinedCompleter
|
||||
elif name == 'FileSearchService':
|
||||
from .at_mentions import FileSearchService
|
||||
return FileSearchService
|
||||
elif name == 'CompareHandler':
|
||||
from .compare import CompareHandler
|
||||
return CompareHandler
|
||||
elif name == 'EvalHandler':
|
||||
from .eval import EvalHandler
|
||||
return EvalHandler
|
||||
elif name == 'SkillsHandler':
|
||||
from .skills import SkillsHandler
|
||||
return SkillsHandler
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -0,0 +1,734 @@
|
||||
"""
|
||||
Agent Scheduler CLI Feature.
|
||||
|
||||
Provides CLI commands for scheduling agents to run 24/7 at regular intervals.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AgentSchedulerHandler:
|
||||
"""Handler for agent scheduler CLI commands."""
|
||||
|
||||
@staticmethod
|
||||
def handle_daemon_command(subcommand: str, args, unknown_args=None) -> int:
|
||||
"""
|
||||
Handle daemon management commands (start, list, stop, logs, restart).
|
||||
|
||||
Args:
|
||||
subcommand: Command to execute (start, list, stop, logs, restart)
|
||||
args: Parsed command-line arguments
|
||||
unknown_args: Additional arguments
|
||||
|
||||
Returns:
|
||||
Exit code
|
||||
"""
|
||||
from praisonai.scheduler.state_manager import SchedulerStateManager
|
||||
from praisonai.scheduler.daemon_manager import DaemonManager
|
||||
|
||||
state_manager = SchedulerStateManager()
|
||||
daemon_manager = DaemonManager()
|
||||
|
||||
if subcommand == "start":
|
||||
return AgentSchedulerHandler._handle_start(args, unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "list":
|
||||
return AgentSchedulerHandler._handle_list(state_manager)
|
||||
elif subcommand == "stop":
|
||||
return AgentSchedulerHandler._handle_stop(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "logs":
|
||||
return AgentSchedulerHandler._handle_logs(unknown_args, daemon_manager)
|
||||
elif subcommand == "restart":
|
||||
return AgentSchedulerHandler._handle_restart(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "delete":
|
||||
return AgentSchedulerHandler._handle_delete(unknown_args, state_manager)
|
||||
elif subcommand == "describe":
|
||||
return AgentSchedulerHandler._handle_describe(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "save":
|
||||
return AgentSchedulerHandler._handle_save(unknown_args, state_manager)
|
||||
elif subcommand == "stop-all":
|
||||
return AgentSchedulerHandler._handle_stop_all(state_manager, daemon_manager)
|
||||
elif subcommand == "stats":
|
||||
return AgentSchedulerHandler._handle_stats(state_manager, unknown_args, daemon_manager)
|
||||
else:
|
||||
print(f"❌ Unknown subcommand: {subcommand}")
|
||||
print("\nAvailable commands:")
|
||||
print(" start, list, stop, logs, restart, delete, describe, save, stop-all, stats")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_start(args, unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule start' command."""
|
||||
from datetime import datetime
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name and task")
|
||||
print("\nUsage:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
print("\nExample:")
|
||||
print(' praisonai schedule start news-checker "Check AI news" --interval hourly')
|
||||
return 1
|
||||
|
||||
# Parse arguments
|
||||
name = unknown_args[0]
|
||||
task = unknown_args[1] if len(unknown_args) > 1 else None
|
||||
|
||||
if not task:
|
||||
print("❌ Error: Please provide a task")
|
||||
return 1
|
||||
|
||||
# Get options
|
||||
interval = getattr(args, 'schedule_interval', None) or 'hourly'
|
||||
max_retries = getattr(args, 'schedule_max_retries', None) or 3
|
||||
timeout = getattr(args, 'timeout', None)
|
||||
max_cost = getattr(args, 'max_cost', None)
|
||||
|
||||
# Check if name already exists
|
||||
existing = state_manager.load_state(name)
|
||||
if existing and daemon_manager.get_status(existing.get('pid', 0))['is_alive']:
|
||||
print(f"❌ Error: Scheduler '{name}' is already running (PID: {existing['pid']})")
|
||||
print(f" Use 'praisonai schedule stop {name}' to stop it first")
|
||||
return 1
|
||||
|
||||
# Start daemon
|
||||
print(f"🚀 Starting scheduler '{name}'...")
|
||||
print(f" Task: {task}")
|
||||
print(f" Interval: {interval}")
|
||||
if timeout:
|
||||
print(f" Timeout: {timeout}s")
|
||||
if max_cost:
|
||||
print(f" Budget: ${max_cost}")
|
||||
|
||||
pid = daemon_manager.start_scheduler_daemon(
|
||||
name=name,
|
||||
task=task,
|
||||
interval=interval,
|
||||
max_cost=max_cost,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries
|
||||
)
|
||||
|
||||
# Save state
|
||||
state = {
|
||||
"name": name,
|
||||
"pid": pid,
|
||||
"task": task,
|
||||
"interval": interval,
|
||||
"timeout": timeout,
|
||||
"max_cost": max_cost,
|
||||
"max_retries": max_retries,
|
||||
"status": "running",
|
||||
"started_at": datetime.now().isoformat(),
|
||||
"executions": 0,
|
||||
"cost": 0.0
|
||||
}
|
||||
state_manager.save_state(name, state)
|
||||
|
||||
print(f"\n✅ Scheduler '{name}' started successfully!")
|
||||
print(f" PID: {pid}")
|
||||
print(f" Logs: ~/.praisonai/logs/{name}.log")
|
||||
print(f"\nManage:")
|
||||
print(f" praisonai schedule list")
|
||||
print(f" praisonai schedule describe {name}")
|
||||
print(f" praisonai schedule logs {name} --follow")
|
||||
print(f" praisonai schedule stop {name}")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_list(state_manager) -> int:
|
||||
"""Handle 'schedule list' command."""
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers running.")
|
||||
print("\nStart one with:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
return 0
|
||||
|
||||
# Clean up dead processes
|
||||
state_manager.cleanup_dead_processes()
|
||||
states = state_manager.list_all()
|
||||
|
||||
# Display table
|
||||
print(f"\n{'Name':<20} {'Status':<10} {'PID':<8} {'Interval':<12} {'Task':<40}")
|
||||
print("=" * 90)
|
||||
|
||||
for state in states:
|
||||
name = state.get('name', 'unknown')[:20]
|
||||
pid = state.get('pid', 0)
|
||||
status = "running" if state_manager.is_process_alive(pid) else "stopped"
|
||||
interval = state.get('interval', 'unknown')[:12]
|
||||
task = state.get('task', '')[:40]
|
||||
|
||||
status_icon = "🟢" if status == "running" else "🔴"
|
||||
print(f"{name:<20} {status_icon} {status:<8} {pid:<8} {interval:<12} {task:<40}")
|
||||
|
||||
print(f"\nTotal: {len(states)} scheduler(s)")
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_stats(state_manager, unknown_args=None, daemon_manager=None) -> int:
|
||||
"""Handle 'schedule stats' command - show aggregate or individual statistics."""
|
||||
# If a name is provided, show individual stats (alias for describe)
|
||||
if unknown_args and len(unknown_args) > 0:
|
||||
name = unknown_args[0]
|
||||
return AgentSchedulerHandler._handle_describe([name], state_manager, daemon_manager)
|
||||
|
||||
# Otherwise show aggregate stats
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers found.")
|
||||
print("\nStart one with:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
return 0
|
||||
|
||||
# Calculate aggregate stats
|
||||
total_schedulers = len(states)
|
||||
running = sum(1 for s in states if s.get('status') == 'running')
|
||||
stopped = total_schedulers - running
|
||||
total_executions = sum(s.get('executions', 0) for s in states)
|
||||
total_cost = sum(s.get('cost', 0.0) for s in states)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"📊 Aggregate Scheduler Statistics")
|
||||
print(f"{'='*60}")
|
||||
print(f"Total Schedulers: {total_schedulers}")
|
||||
print(f" 🟢 Running: {running}")
|
||||
print(f" 🔴 Stopped: {stopped}")
|
||||
print(f"\nTotal Executions: {total_executions}")
|
||||
print(f"Total Cost: ${total_cost:.4f}")
|
||||
|
||||
if total_executions > 0:
|
||||
avg_cost = total_cost / total_executions
|
||||
print(f"Avg Cost/Execution: ${avg_cost:.4f}")
|
||||
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Show per-scheduler breakdown
|
||||
print("Per-Scheduler Breakdown:")
|
||||
print(f"{'Name':<20} {'Status':<10} {'Executions':<12} {'Cost':<12}")
|
||||
print("="*60)
|
||||
|
||||
for state in sorted(states, key=lambda x: x.get('executions', 0), reverse=True):
|
||||
name = state['name'][:20]
|
||||
status = state.get('status', 'unknown')
|
||||
executions = state.get('executions', 0)
|
||||
cost = state.get('cost', 0.0)
|
||||
|
||||
status_icon = "🟢" if status == "running" else "🔴"
|
||||
print(f"{name:<20} {status_icon} {status:<8} {executions:<12} ${cost:<11.4f}")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_stop_all(state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule stop-all' command."""
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers running.")
|
||||
return 0
|
||||
|
||||
print(f"\n🛑 Stopping all schedulers ({len(states)} total)...\n")
|
||||
|
||||
stopped = 0
|
||||
failed = 0
|
||||
|
||||
for state in states:
|
||||
name = state['name']
|
||||
pid = state['pid']
|
||||
|
||||
try:
|
||||
if daemon_manager.stop_daemon(pid):
|
||||
state_manager.delete_state(name)
|
||||
print(f"✅ Stopped '{name}' (PID: {pid})")
|
||||
stopped += 1
|
||||
else:
|
||||
print(f"❌ Failed to stop '{name}' (PID: {pid})")
|
||||
failed += 1
|
||||
except Exception as e:
|
||||
print(f"❌ Error stopping '{name}': {e}")
|
||||
failed += 1
|
||||
|
||||
print(f"\n📊 Summary: {stopped} stopped, {failed} failed")
|
||||
return 0 if failed == 0 else 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_stop(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule stop' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule stop <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
print("\nList schedulers with: praisonai schedule list")
|
||||
return 1
|
||||
|
||||
pid = state.get('pid')
|
||||
if not pid:
|
||||
print(f"❌ Error: No PID found for scheduler '{name}'")
|
||||
return 1
|
||||
|
||||
print(f"🛑 Stopping scheduler '{name}' (PID: {pid})...")
|
||||
|
||||
success = daemon_manager.stop_daemon(pid)
|
||||
|
||||
if success:
|
||||
state['status'] = 'stopped'
|
||||
state_manager.save_state(name, state)
|
||||
print(f"✅ Scheduler '{name}' stopped successfully")
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Failed to stop scheduler '{name}'")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_logs(unknown_args, daemon_manager) -> int:
|
||||
"""Handle 'schedule logs' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule logs <name> [-f]")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
follow = '-f' in unknown_args or '--follow' in unknown_args
|
||||
|
||||
if follow:
|
||||
print(f"📋 Following logs for '{name}' (Ctrl+C to stop)...")
|
||||
import subprocess
|
||||
log_file = daemon_manager.log_dir / f"{name}.log"
|
||||
if not log_file.exists():
|
||||
print(f"❌ Error: Log file not found for '{name}'")
|
||||
return 1
|
||||
|
||||
try:
|
||||
subprocess.run(['tail', '-f', str(log_file)])
|
||||
except KeyboardInterrupt:
|
||||
print("\n✅ Stopped following logs")
|
||||
return 0
|
||||
else:
|
||||
logs = daemon_manager.read_logs(name, lines=50)
|
||||
if logs:
|
||||
print(f"📋 Last 50 lines of logs for '{name}':\n")
|
||||
print(logs)
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Error: No logs found for '{name}'")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_restart(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule restart' command."""
|
||||
from datetime import datetime
|
||||
import time
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule restart <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
print(f"🔄 Restarting scheduler '{name}'...")
|
||||
|
||||
# Stop if running
|
||||
pid = state.get('pid')
|
||||
if pid and state_manager.is_process_alive(pid):
|
||||
daemon_manager.stop_daemon(pid)
|
||||
time.sleep(1)
|
||||
|
||||
# Start again
|
||||
new_pid = daemon_manager.start_scheduler_daemon(
|
||||
name=name,
|
||||
task=state['task'],
|
||||
interval=state['interval'],
|
||||
max_cost=state.get('max_cost'),
|
||||
timeout=state.get('timeout'),
|
||||
max_retries=state.get('max_retries', 3)
|
||||
)
|
||||
|
||||
state['pid'] = new_pid
|
||||
state['status'] = 'running'
|
||||
state['started_at'] = datetime.now().isoformat()
|
||||
state_manager.save_state(name, state)
|
||||
|
||||
print(f"✅ Scheduler '{name}' restarted (PID: {new_pid})")
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_delete(unknown_args, state_manager) -> int:
|
||||
"""Handle 'schedule delete' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule delete <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
|
||||
if state_manager.delete_state(name):
|
||||
print(f"✅ Scheduler '{name}' deleted from list")
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_describe(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule describe' command."""
|
||||
from datetime import datetime
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule describe <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
print("\nList schedulers with: praisonai schedule list")
|
||||
return 1
|
||||
|
||||
# Get process status
|
||||
pid = state.get('pid', 0)
|
||||
is_alive = state_manager.is_process_alive(pid)
|
||||
status = "🟢 running" if is_alive else "🔴 stopped"
|
||||
|
||||
# Calculate uptime
|
||||
started_at = state.get('started_at')
|
||||
uptime = "N/A"
|
||||
if started_at:
|
||||
try:
|
||||
start_time = datetime.fromisoformat(started_at)
|
||||
uptime_delta = datetime.now() - start_time
|
||||
hours = int(uptime_delta.total_seconds() // 3600)
|
||||
minutes = int((uptime_delta.total_seconds() % 3600) // 60)
|
||||
uptime = f"{hours}h {minutes}m"
|
||||
except:
|
||||
pass
|
||||
|
||||
# Display detailed info
|
||||
print(f"\n{'='*60}")
|
||||
print(f"📋 Scheduler Details: {name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"Status: {status}")
|
||||
print(f"PID: {pid}")
|
||||
print(f"Uptime: {uptime}")
|
||||
print(f"Task: {state.get('task', 'N/A')}")
|
||||
print(f"Interval: {state.get('interval', 'N/A')}")
|
||||
print(f"Max Retries: {state.get('max_retries', 'N/A')}")
|
||||
|
||||
if state.get('timeout'):
|
||||
print(f"Timeout: {state['timeout']}s")
|
||||
|
||||
if state.get('max_cost'):
|
||||
print(f"Budget: ${state['max_cost']}")
|
||||
|
||||
print(f"Executions: {state.get('executions', 0)}")
|
||||
print(f"Total Cost: ${state.get('cost', 0.0):.4f}")
|
||||
print(f"Started: {started_at or 'N/A'}")
|
||||
|
||||
# Log file location
|
||||
log_file = daemon_manager.log_dir / f"{name}.log"
|
||||
print(f"Logs: {log_file}")
|
||||
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_save(unknown_args, state_manager) -> int:
|
||||
"""Handle 'schedule save' command."""
|
||||
import yaml
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule save <name> [output.yaml]")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
output_file = unknown_args[1] if len(unknown_args) > 1 else f"{name}.yaml"
|
||||
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
# Create YAML config from state
|
||||
yaml_config = {
|
||||
'framework': 'praisonai',
|
||||
'agents': [{
|
||||
'name': name,
|
||||
'role': 'Task Executor',
|
||||
'goal': state.get('task', ''),
|
||||
'instructions': state.get('task', ''),
|
||||
'verbose': True
|
||||
}],
|
||||
'task': state.get('task', ''),
|
||||
'schedule': {
|
||||
'interval': state.get('interval', 'hourly'),
|
||||
'max_retries': state.get('max_retries', 3),
|
||||
'run_immediately': True
|
||||
}
|
||||
}
|
||||
|
||||
# Add optional fields
|
||||
if state.get('timeout'):
|
||||
yaml_config['schedule']['timeout'] = state['timeout']
|
||||
|
||||
if state.get('max_cost'):
|
||||
yaml_config['schedule']['max_cost'] = state['max_cost']
|
||||
|
||||
# Write to file
|
||||
try:
|
||||
with open(output_file, 'w') as f:
|
||||
yaml.dump(yaml_config, f, default_flow_style=False, sort_keys=False)
|
||||
|
||||
print(f"✅ Configuration saved to: {output_file}")
|
||||
print(f"\nRun with:")
|
||||
print(f" praisonai schedule {output_file}")
|
||||
return 0
|
||||
except Exception as e:
|
||||
print(f"❌ Error saving configuration: {e}")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def handle_schedule_command(args, unknown_args, daemon_mode=False) -> int:
|
||||
"""
|
||||
Handle the schedule command for running agents periodically.
|
||||
|
||||
Supports two modes:
|
||||
1. YAML mode: praisonai schedule agents.yaml
|
||||
2. Prompt mode: praisonai schedule "Your task here" --interval hourly
|
||||
|
||||
Args:
|
||||
args: Parsed command-line arguments
|
||||
unknown_args: Additional arguments after the command
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, 1 for error)
|
||||
"""
|
||||
try:
|
||||
from praisonai.scheduler import AgentScheduler
|
||||
from praisonaiagents import Agent
|
||||
except ImportError:
|
||||
print("Error: praisonai.scheduler or praisonaiagents module not found")
|
||||
print("Please ensure PraisonAI is properly installed")
|
||||
print("pip install praisonai praisonaiagents")
|
||||
return 1
|
||||
|
||||
# Check if first arg is a YAML file or a prompt
|
||||
first_arg = unknown_args[0] if unknown_args else None
|
||||
is_yaml_mode = first_arg and (first_arg.endswith('.yaml') or first_arg.endswith('.yml'))
|
||||
|
||||
# Get overrides from CLI
|
||||
interval_override = getattr(args, 'schedule_interval', None) or 'hourly'
|
||||
max_retries_override = getattr(args, 'schedule_max_retries', None) or 3
|
||||
timeout_override = getattr(args, 'timeout', None)
|
||||
max_cost_override = getattr(args, 'max_cost', None)
|
||||
verbose = getattr(args, 'verbose', False)
|
||||
|
||||
# Set up logging - only show logs if verbose
|
||||
if verbose:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='[%(asctime)s] %(levelname)s - %(message)s',
|
||||
datefmt='%Y-%m-%d %H:%M:%S'
|
||||
)
|
||||
else:
|
||||
# Suppress scheduler logs in non-verbose mode
|
||||
logging.getLogger('praisonai.scheduler').setLevel(logging.WARNING)
|
||||
|
||||
try:
|
||||
if is_yaml_mode:
|
||||
# YAML mode: Load from agents.yaml
|
||||
yaml_path = first_arg
|
||||
print(f"🤖 Loading agent configuration from: {yaml_path}")
|
||||
scheduler = AgentScheduler.from_yaml(
|
||||
yaml_path=yaml_path,
|
||||
interval_override=interval_override,
|
||||
max_retries_override=max_retries_override,
|
||||
timeout_override=timeout_override,
|
||||
max_cost_override=max_cost_override
|
||||
)
|
||||
else:
|
||||
# Prompt mode: Create agent from direct prompt
|
||||
if not first_arg:
|
||||
print("❌ Error: Please provide either a YAML file or a task prompt")
|
||||
print("\nExamples:")
|
||||
print(" praisonai schedule agents.yaml")
|
||||
print(' praisonai schedule "Check news every hour" --interval hourly')
|
||||
return 1
|
||||
|
||||
task = first_arg
|
||||
|
||||
# Create agent
|
||||
agent = Agent(
|
||||
name="Scheduled Agent",
|
||||
role="Task Executor",
|
||||
goal=task,
|
||||
instructions=task,
|
||||
verbose=True # Enable verbose to see output in logs
|
||||
)
|
||||
|
||||
# Create scheduler
|
||||
scheduler = AgentScheduler(
|
||||
agent=agent,
|
||||
task=task,
|
||||
timeout=timeout_override,
|
||||
max_cost=max_cost_override
|
||||
)
|
||||
|
||||
# Get configuration
|
||||
interval = interval_override
|
||||
max_retries = max_retries_override
|
||||
|
||||
if is_yaml_mode:
|
||||
schedule_config = scheduler._yaml_schedule_config
|
||||
interval = interval_override or schedule_config.get('interval', 'hourly')
|
||||
max_retries = max_retries_override or schedule_config.get('max_retries', 3)
|
||||
|
||||
print(f"Schedule: {interval}")
|
||||
print(f"Max Retries: {max_retries}")
|
||||
if scheduler.timeout:
|
||||
print(f"Timeout: {scheduler.timeout}s")
|
||||
if scheduler.max_cost:
|
||||
print(f"Budget: ${scheduler.max_cost}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Start scheduler
|
||||
if is_yaml_mode:
|
||||
scheduler.start_from_yaml_config()
|
||||
else:
|
||||
scheduler.start(schedule_expr=interval, max_retries=max_retries, run_immediately=True)
|
||||
|
||||
# If daemon mode, block to keep process alive
|
||||
if daemon_mode:
|
||||
import time
|
||||
try:
|
||||
while scheduler.is_running:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
scheduler.stop()
|
||||
else:
|
||||
# Foreground mode - wait for Ctrl+C
|
||||
print("⏰ Starting scheduler... (Press Ctrl+C to stop)\n")
|
||||
import time
|
||||
try:
|
||||
while scheduler.is_running:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n🛑 Stopping scheduler...")
|
||||
scheduler.stop()
|
||||
|
||||
# Display final stats
|
||||
stats = scheduler.get_stats()
|
||||
print("\n📊 Final Statistics:")
|
||||
print(f" Total Executions: {stats['total_executions']}")
|
||||
print(f" Successful: {stats['successful_executions']}")
|
||||
print(f" Failed: {stats['failed_executions']}")
|
||||
print(f" Success Rate: {stats['success_rate']:.1f}%")
|
||||
print("\n✅ Agent stopped successfully\n")
|
||||
|
||||
return 0
|
||||
|
||||
except FileNotFoundError as e:
|
||||
print(f"❌ Error: {e}")
|
||||
print(f"\nMake sure {yaml_path} exists in the current directory.")
|
||||
print("\nExample agents.yaml structure:")
|
||||
print("""
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "AI News Monitor"
|
||||
role: "News Analyst"
|
||||
instructions: "Search and summarize AI news"
|
||||
tools:
|
||||
- search_tool
|
||||
|
||||
task: "Search for latest AI news"
|
||||
|
||||
schedule:
|
||||
interval: "hourly"
|
||||
max_retries: 3
|
||||
run_immediately: true
|
||||
""")
|
||||
return 1
|
||||
|
||||
except ValueError as e:
|
||||
print(f"❌ Configuration Error: {e}")
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {e}")
|
||||
if verbose:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def add_schedule_arguments(subparsers):
|
||||
"""
|
||||
Add schedule subcommand and arguments to parser.
|
||||
|
||||
Args:
|
||||
subparsers: Subparsers object from argparse
|
||||
"""
|
||||
schedule_parser = subparsers.add_parser(
|
||||
'schedule',
|
||||
help='Schedule an agent to run continuously at regular intervals'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'schedule_yaml',
|
||||
nargs='?',
|
||||
default='agents.yaml',
|
||||
help='Path to agents.yaml file (default: agents.yaml)'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--interval',
|
||||
dest='schedule_interval',
|
||||
type=str,
|
||||
help='Override schedule interval (e.g., "hourly", "*/30m", "daily")'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--max-retries',
|
||||
dest='schedule_max_retries',
|
||||
type=int,
|
||||
help='Override maximum retry attempts (default: from YAML or 3)'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--verbose', '-v',
|
||||
action='store_true',
|
||||
help='Enable verbose logging'
|
||||
)
|
||||
|
||||
schedule_parser.set_defaults(func=AgentSchedulerHandler.handle_schedule_command)
|
||||
|
||||
|
||||
def setup_scheduler_cli(subparsers):
|
||||
"""
|
||||
Setup scheduler CLI commands.
|
||||
|
||||
Args:
|
||||
subparsers: Subparsers object from argparse
|
||||
"""
|
||||
AgentSchedulerHandler.add_schedule_arguments(subparsers)
|
||||
@@ -0,0 +1,729 @@
|
||||
"""
|
||||
Agent Scheduler CLI Feature.
|
||||
|
||||
Provides CLI commands for scheduling agents to run 24/7 at regular intervals.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AgentSchedulerHandler:
|
||||
"""Handler for agent scheduler CLI commands."""
|
||||
|
||||
@staticmethod
|
||||
def handle_daemon_command(subcommand: str, args, unknown_args=None) -> int:
|
||||
"""
|
||||
Handle daemon management commands (start, list, stop, logs, restart).
|
||||
|
||||
Args:
|
||||
subcommand: Command to execute (start, list, stop, logs, restart)
|
||||
args: Parsed command-line arguments
|
||||
unknown_args: Additional arguments
|
||||
|
||||
Returns:
|
||||
Exit code
|
||||
"""
|
||||
from praisonai.scheduler.state_manager import SchedulerStateManager
|
||||
from praisonai.scheduler.daemon_manager import DaemonManager
|
||||
|
||||
state_manager = SchedulerStateManager()
|
||||
daemon_manager = DaemonManager()
|
||||
|
||||
if subcommand == "start":
|
||||
return AgentSchedulerHandler._handle_start(args, unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "list":
|
||||
return AgentSchedulerHandler._handle_list(state_manager)
|
||||
elif subcommand == "stop":
|
||||
return AgentSchedulerHandler._handle_stop(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "logs":
|
||||
return AgentSchedulerHandler._handle_logs(unknown_args, daemon_manager)
|
||||
elif subcommand == "restart":
|
||||
return AgentSchedulerHandler._handle_restart(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "delete":
|
||||
return AgentSchedulerHandler._handle_delete(unknown_args, state_manager)
|
||||
elif subcommand == "describe":
|
||||
return AgentSchedulerHandler._handle_describe(unknown_args, state_manager, daemon_manager)
|
||||
elif subcommand == "save":
|
||||
return AgentSchedulerHandler._handle_save(unknown_args, state_manager)
|
||||
elif subcommand == "stop-all":
|
||||
return AgentSchedulerHandler._handle_stop_all(state_manager, daemon_manager)
|
||||
elif subcommand == "stats":
|
||||
return AgentSchedulerHandler._handle_stats(state_manager, unknown_args, daemon_manager)
|
||||
else:
|
||||
print(f"❌ Unknown subcommand: {subcommand}")
|
||||
print("\nAvailable commands:")
|
||||
print(" start, list, stop, logs, restart, delete, describe, save, stop-all, stats")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_start(args, unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule start' command."""
|
||||
from datetime import datetime
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name and task")
|
||||
print("\nUsage:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
print("\nExample:")
|
||||
print(' praisonai schedule start news-checker "Check AI news" --interval hourly')
|
||||
return 1
|
||||
|
||||
# Parse arguments
|
||||
name = unknown_args[0]
|
||||
task = unknown_args[1] if len(unknown_args) > 1 else None
|
||||
|
||||
if not task:
|
||||
print("❌ Error: Please provide a task")
|
||||
return 1
|
||||
|
||||
# Get options
|
||||
interval = getattr(args, 'schedule_interval', None) or 'hourly'
|
||||
max_retries = getattr(args, 'schedule_max_retries', None) or 3
|
||||
timeout = getattr(args, 'timeout', None)
|
||||
max_cost = getattr(args, 'max_cost', None)
|
||||
|
||||
# Check if name already exists
|
||||
existing = state_manager.load_state(name)
|
||||
if existing and daemon_manager.get_status(existing.get('pid', 0))['is_alive']:
|
||||
print(f"❌ Error: Scheduler '{name}' is already running (PID: {existing['pid']})")
|
||||
print(f" Use 'praisonai schedule stop {name}' to stop it first")
|
||||
return 1
|
||||
|
||||
# Start daemon
|
||||
print(f"🚀 Starting scheduler '{name}'...")
|
||||
print(f" Task: {task}")
|
||||
print(f" Interval: {interval}")
|
||||
if timeout:
|
||||
print(f" Timeout: {timeout}s")
|
||||
if max_cost:
|
||||
print(f" Budget: ${max_cost}")
|
||||
|
||||
pid = daemon_manager.start_scheduler_daemon(
|
||||
name=name,
|
||||
task=task,
|
||||
interval=interval,
|
||||
max_cost=max_cost,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries
|
||||
)
|
||||
|
||||
# Save state
|
||||
state = {
|
||||
"name": name,
|
||||
"pid": pid,
|
||||
"task": task,
|
||||
"interval": interval,
|
||||
"timeout": timeout,
|
||||
"max_cost": max_cost,
|
||||
"max_retries": max_retries,
|
||||
"status": "running",
|
||||
"started_at": datetime.now().isoformat(),
|
||||
"executions": 0,
|
||||
"cost": 0.0
|
||||
}
|
||||
state_manager.save_state(name, state)
|
||||
|
||||
print(f"\n✅ Scheduler '{name}' started successfully!")
|
||||
print(f" PID: {pid}")
|
||||
print(f" Logs: ~/.praisonai/logs/{name}.log")
|
||||
print(f"\nManage:")
|
||||
print(f" praisonai schedule list")
|
||||
print(f" praisonai schedule describe {name}")
|
||||
print(f" praisonai schedule logs {name} --follow")
|
||||
print(f" praisonai schedule stop {name}")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_list(state_manager) -> int:
|
||||
"""Handle 'schedule list' command."""
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers running.")
|
||||
print("\nStart one with:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
return 0
|
||||
|
||||
# Clean up dead processes
|
||||
state_manager.cleanup_dead_processes()
|
||||
states = state_manager.list_all()
|
||||
|
||||
# Display table
|
||||
print(f"\n{'Name':<20} {'Status':<10} {'PID':<8} {'Interval':<12} {'Task':<40}")
|
||||
print("=" * 90)
|
||||
|
||||
for state in states:
|
||||
name = state.get('name', 'unknown')[:20]
|
||||
pid = state.get('pid', 0)
|
||||
status = "running" if state_manager.is_process_alive(pid) else "stopped"
|
||||
interval = state.get('interval', 'unknown')[:12]
|
||||
task = state.get('task', '')[:40]
|
||||
|
||||
status_icon = "🟢" if status == "running" else "🔴"
|
||||
print(f"{name:<20} {status_icon} {status:<8} {pid:<8} {interval:<12} {task:<40}")
|
||||
|
||||
print(f"\nTotal: {len(states)} scheduler(s)")
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_stats(state_manager, unknown_args=None, daemon_manager=None) -> int:
|
||||
"""Handle 'schedule stats' command - show aggregate or individual statistics."""
|
||||
# If a name is provided, show individual stats (alias for describe)
|
||||
if unknown_args and len(unknown_args) > 0:
|
||||
name = unknown_args[0]
|
||||
return AgentSchedulerHandler._handle_describe([name], state_manager, None)
|
||||
|
||||
# Otherwise show aggregate stats
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers found.")
|
||||
print("\nStart one with:")
|
||||
print(' praisonai schedule start <name> "Your task" --interval hourly')
|
||||
return 0
|
||||
|
||||
# Calculate aggregate stats
|
||||
total_schedulers = len(states)
|
||||
running = sum(1 for s in states if s.get('status') == 'running')
|
||||
stopped = total_schedulers - running
|
||||
total_executions = sum(s.get('executions', 0) for s in states)
|
||||
total_cost = sum(s.get('cost', 0.0) for s in states)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"📊 Aggregate Scheduler Statistics")
|
||||
print(f"{'='*60}")
|
||||
print(f"Total Schedulers: {total_schedulers}")
|
||||
print(f" 🟢 Running: {running}")
|
||||
print(f" 🔴 Stopped: {stopped}")
|
||||
print(f"\nTotal Executions: {total_executions}")
|
||||
print(f"Total Cost: ${total_cost:.4f}")
|
||||
|
||||
if total_executions > 0:
|
||||
avg_cost = total_cost / total_executions
|
||||
print(f"Avg Cost/Execution: ${avg_cost:.4f}")
|
||||
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Show per-scheduler breakdown
|
||||
print("Per-Scheduler Breakdown:")
|
||||
print(f"{'Name':<20} {'Status':<10} {'Executions':<12} {'Cost':<12}")
|
||||
print("="*60)
|
||||
|
||||
for state in sorted(states, key=lambda x: x.get('executions', 0), reverse=True):
|
||||
name = state['name'][:20]
|
||||
status = state.get('status', 'unknown')
|
||||
executions = state.get('executions', 0)
|
||||
cost = state.get('cost', 0.0)
|
||||
|
||||
status_icon = "🟢" if status == "running" else "🔴"
|
||||
print(f"{name:<20} {status_icon} {status:<8} {executions:<12} ${cost:<11.4f}")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_stop_all(state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule stop-all' command."""
|
||||
states = state_manager.list_all()
|
||||
|
||||
if not states:
|
||||
print("No schedulers running.")
|
||||
return 0
|
||||
|
||||
print(f"\n🛑 Stopping all schedulers ({len(states)} total)...\n")
|
||||
|
||||
stopped = 0
|
||||
failed = 0
|
||||
|
||||
for state in states:
|
||||
name = state['name']
|
||||
pid = state['pid']
|
||||
|
||||
try:
|
||||
if daemon_manager.stop_daemon(pid):
|
||||
state_manager.delete_state(name)
|
||||
print(f"✅ Stopped '{name}' (PID: {pid})")
|
||||
stopped += 1
|
||||
else:
|
||||
print(f"❌ Failed to stop '{name}' (PID: {pid})")
|
||||
failed += 1
|
||||
except Exception as e:
|
||||
print(f"❌ Error stopping '{name}': {e}")
|
||||
failed += 1
|
||||
|
||||
print(f"\n📊 Summary: {stopped} stopped, {failed} failed")
|
||||
return 0 if failed == 0 else 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_stop(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule stop' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule stop <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
print("\nList schedulers with: praisonai schedule list")
|
||||
return 1
|
||||
|
||||
pid = state.get('pid')
|
||||
if not pid:
|
||||
print(f"❌ Error: No PID found for scheduler '{name}'")
|
||||
return 1
|
||||
|
||||
print(f"🛑 Stopping scheduler '{name}' (PID: {pid})...")
|
||||
|
||||
success = daemon_manager.stop_daemon(pid)
|
||||
|
||||
if success:
|
||||
state['status'] = 'stopped'
|
||||
state_manager.save_state(name, state)
|
||||
print(f"✅ Scheduler '{name}' stopped successfully")
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Failed to stop scheduler '{name}'")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_logs(unknown_args, daemon_manager) -> int:
|
||||
"""Handle 'schedule logs' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule logs <name> [-f]")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
follow = '-f' in unknown_args or '--follow' in unknown_args
|
||||
|
||||
if follow:
|
||||
print(f"📋 Following logs for '{name}' (Ctrl+C to stop)...")
|
||||
import subprocess
|
||||
log_file = daemon_manager.log_dir / f"{name}.log"
|
||||
if not log_file.exists():
|
||||
print(f"❌ Error: Log file not found for '{name}'")
|
||||
return 1
|
||||
|
||||
try:
|
||||
subprocess.run(['tail', '-f', str(log_file)])
|
||||
except KeyboardInterrupt:
|
||||
print("\n✅ Stopped following logs")
|
||||
return 0
|
||||
else:
|
||||
logs = daemon_manager.read_logs(name, lines=50)
|
||||
if logs:
|
||||
print(f"📋 Last 50 lines of logs for '{name}':\n")
|
||||
print(logs)
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Error: No logs found for '{name}'")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_restart(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule restart' command."""
|
||||
from datetime import datetime
|
||||
import time
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule restart <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
print(f"🔄 Restarting scheduler '{name}'...")
|
||||
|
||||
# Stop if running
|
||||
pid = state.get('pid')
|
||||
if pid and state_manager.is_process_alive(pid):
|
||||
daemon_manager.stop_daemon(pid)
|
||||
time.sleep(1)
|
||||
|
||||
# Start again
|
||||
new_pid = daemon_manager.start_scheduler_daemon(
|
||||
name=name,
|
||||
task=state['task'],
|
||||
interval=state['interval'],
|
||||
max_cost=state.get('max_cost'),
|
||||
timeout=state.get('timeout'),
|
||||
max_retries=state.get('max_retries', 3)
|
||||
)
|
||||
|
||||
state['pid'] = new_pid
|
||||
state['status'] = 'running'
|
||||
state['started_at'] = datetime.now().isoformat()
|
||||
state_manager.save_state(name, state)
|
||||
|
||||
print(f"✅ Scheduler '{name}' restarted (PID: {new_pid})")
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_delete(unknown_args, state_manager) -> int:
|
||||
"""Handle 'schedule delete' command."""
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule delete <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
|
||||
if state_manager.delete_state(name):
|
||||
print(f"✅ Scheduler '{name}' deleted from list")
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _handle_describe(unknown_args, state_manager, daemon_manager) -> int:
|
||||
"""Handle 'schedule describe' command."""
|
||||
from datetime import datetime
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule describe <name>")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
print("\nList schedulers with: praisonai schedule list")
|
||||
return 1
|
||||
|
||||
# Get process status
|
||||
pid = state.get('pid', 0)
|
||||
is_alive = state_manager.is_process_alive(pid)
|
||||
status = "🟢 running" if is_alive else "🔴 stopped"
|
||||
|
||||
# Calculate uptime
|
||||
started_at = state.get('started_at')
|
||||
uptime = "N/A"
|
||||
if started_at:
|
||||
try:
|
||||
start_time = datetime.fromisoformat(started_at)
|
||||
uptime_delta = datetime.now() - start_time
|
||||
hours = int(uptime_delta.total_seconds() // 3600)
|
||||
minutes = int((uptime_delta.total_seconds() % 3600) // 60)
|
||||
uptime = f"{hours}h {minutes}m"
|
||||
except:
|
||||
pass
|
||||
|
||||
# Display detailed info
|
||||
print(f"\n{'='*60}")
|
||||
print(f"📋 Scheduler Details: {name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"Status: {status}")
|
||||
print(f"PID: {pid}")
|
||||
print(f"Uptime: {uptime}")
|
||||
print(f"Task: {state.get('task', 'N/A')}")
|
||||
print(f"Interval: {state.get('interval', 'N/A')}")
|
||||
print(f"Max Retries: {state.get('max_retries', 'N/A')}")
|
||||
|
||||
if state.get('timeout'):
|
||||
print(f"Timeout: {state['timeout']}s")
|
||||
|
||||
if state.get('max_cost'):
|
||||
print(f"Budget: ${state['max_cost']}")
|
||||
|
||||
print(f"Executions: {state.get('executions', 0)}")
|
||||
print(f"Total Cost: ${state.get('cost', 0.0):.4f}")
|
||||
print(f"Started: {started_at or 'N/A'}")
|
||||
|
||||
# Log file location
|
||||
log_file = daemon_manager.log_dir / f"{name}.log"
|
||||
print(f"Logs: {log_file}")
|
||||
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _handle_save(unknown_args, state_manager) -> int:
|
||||
"""Handle 'schedule save' command."""
|
||||
import yaml
|
||||
|
||||
if not unknown_args:
|
||||
print("❌ Error: Please provide scheduler name")
|
||||
print("\nUsage: praisonai schedule save <name> [output.yaml]")
|
||||
return 1
|
||||
|
||||
name = unknown_args[0]
|
||||
output_file = unknown_args[1] if len(unknown_args) > 1 else f"{name}.yaml"
|
||||
|
||||
state = state_manager.load_state(name)
|
||||
|
||||
if not state:
|
||||
print(f"❌ Error: Scheduler '{name}' not found")
|
||||
return 1
|
||||
|
||||
# Create YAML config from state
|
||||
yaml_config = {
|
||||
'framework': 'praisonai',
|
||||
'agents': [{
|
||||
'name': name,
|
||||
'role': 'Task Executor',
|
||||
'goal': state.get('task', ''),
|
||||
'instructions': state.get('task', ''),
|
||||
'verbose': True
|
||||
}],
|
||||
'task': state.get('task', ''),
|
||||
'schedule': {
|
||||
'interval': state.get('interval', 'hourly'),
|
||||
'max_retries': state.get('max_retries', 3),
|
||||
'run_immediately': True
|
||||
}
|
||||
}
|
||||
|
||||
# Add optional fields
|
||||
if state.get('timeout'):
|
||||
yaml_config['schedule']['timeout'] = state['timeout']
|
||||
|
||||
if state.get('max_cost'):
|
||||
yaml_config['schedule']['max_cost'] = state['max_cost']
|
||||
|
||||
# Write to file
|
||||
try:
|
||||
with open(output_file, 'w') as f:
|
||||
yaml.dump(yaml_config, f, default_flow_style=False, sort_keys=False)
|
||||
|
||||
print(f"✅ Configuration saved to: {output_file}")
|
||||
print(f"\nRun with:")
|
||||
print(f" praisonai schedule {output_file}")
|
||||
return 0
|
||||
except Exception as e:
|
||||
print(f"❌ Error saving configuration: {e}")
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def handle_schedule_command(args, unknown_args=None) -> int:
|
||||
"""
|
||||
Handle the schedule command for running agents periodically.
|
||||
|
||||
Supports two modes:
|
||||
1. YAML mode: praisonai schedule agents.yaml
|
||||
2. Prompt mode: praisonai schedule "Your task here" --interval hourly
|
||||
|
||||
Args:
|
||||
args: Parsed command-line arguments
|
||||
unknown_args: Additional arguments after the command
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, 1 for error)
|
||||
"""
|
||||
try:
|
||||
from praisonai.scheduler import AgentScheduler
|
||||
from praisonaiagents import Agent
|
||||
except ImportError:
|
||||
print("Error: praisonai.scheduler or praisonaiagents module not found")
|
||||
print("Please ensure PraisonAI is properly installed")
|
||||
print("pip install praisonai praisonaiagents")
|
||||
return 1
|
||||
|
||||
# Check if first arg is a YAML file or a prompt
|
||||
first_arg = unknown_args[0] if unknown_args else None
|
||||
is_yaml_mode = first_arg and (first_arg.endswith('.yaml') or first_arg.endswith('.yml'))
|
||||
|
||||
# Get overrides from CLI
|
||||
interval_override = getattr(args, 'schedule_interval', None) or 'hourly'
|
||||
max_retries_override = getattr(args, 'schedule_max_retries', None) or 3
|
||||
timeout_override = getattr(args, 'timeout', None)
|
||||
max_cost_override = getattr(args, 'max_cost', None)
|
||||
verbose = getattr(args, 'verbose', False)
|
||||
|
||||
# Set up logging - only show logs if verbose
|
||||
if verbose:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='[%(asctime)s] %(levelname)s - %(message)s',
|
||||
datefmt='%Y-%m-%d %H:%M:%S'
|
||||
)
|
||||
else:
|
||||
# Suppress scheduler logs in non-verbose mode
|
||||
logging.getLogger('praisonai.scheduler').setLevel(logging.WARNING)
|
||||
|
||||
try:
|
||||
if is_yaml_mode:
|
||||
# YAML mode: Load from agents.yaml
|
||||
yaml_path = first_arg
|
||||
print(f"🤖 Loading agent configuration from: {yaml_path}")
|
||||
scheduler = AgentScheduler.from_yaml(
|
||||
yaml_path=yaml_path,
|
||||
interval_override=interval_override,
|
||||
max_retries_override=max_retries_override,
|
||||
timeout_override=timeout_override,
|
||||
max_cost_override=max_cost_override
|
||||
)
|
||||
else:
|
||||
# Prompt mode: Create agent from direct prompt
|
||||
if not first_arg:
|
||||
print("❌ Error: Please provide either a YAML file or a task prompt")
|
||||
print("\nExamples:")
|
||||
print(" praisonai schedule agents.yaml")
|
||||
print(' praisonai schedule "Check news every hour" --interval hourly')
|
||||
return 1
|
||||
|
||||
task = first_arg
|
||||
|
||||
# Create agent
|
||||
agent = Agent(
|
||||
name="Scheduled Agent",
|
||||
role="Task Executor",
|
||||
goal=task,
|
||||
instructions=task,
|
||||
verbose=False
|
||||
)
|
||||
|
||||
# Create scheduler
|
||||
scheduler = AgentScheduler(
|
||||
agent=agent,
|
||||
task=task,
|
||||
timeout=timeout_override,
|
||||
max_cost=max_cost_override
|
||||
)
|
||||
|
||||
# Get configuration
|
||||
interval = interval_override
|
||||
max_retries = max_retries_override
|
||||
|
||||
if is_yaml_mode:
|
||||
schedule_config = scheduler._yaml_schedule_config
|
||||
interval = interval_override or schedule_config.get('interval', 'hourly')
|
||||
max_retries = max_retries_override or schedule_config.get('max_retries', 3)
|
||||
|
||||
print(f"Schedule: {interval}")
|
||||
print(f"Max Retries: {max_retries}")
|
||||
if scheduler.timeout:
|
||||
print(f"Timeout: {scheduler.timeout}s")
|
||||
if scheduler.max_cost:
|
||||
print(f"Budget: ${scheduler.max_cost}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Start scheduler
|
||||
print("⏰ Starting scheduler... (Press Ctrl+C to stop)\n")
|
||||
if is_yaml_mode:
|
||||
scheduler.start_from_yaml_config()
|
||||
else:
|
||||
scheduler.start(
|
||||
schedule_expr=interval,
|
||||
max_retries=max_retries,
|
||||
run_immediately=True
|
||||
)
|
||||
|
||||
# Keep running until interrupted
|
||||
try:
|
||||
while scheduler.is_running:
|
||||
import time
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n🛑 Stopping scheduler...")
|
||||
scheduler.stop()
|
||||
|
||||
# Display final stats
|
||||
stats = scheduler.get_stats()
|
||||
print("\n📊 Final Statistics:")
|
||||
print(f" Total Executions: {stats['total_executions']}")
|
||||
print(f" Successful: {stats['successful_executions']}")
|
||||
print(f" Failed: {stats['failed_executions']}")
|
||||
print(f" Success Rate: {stats['success_rate']:.1f}%")
|
||||
print("\n✅ Agent stopped successfully\n")
|
||||
|
||||
return 0
|
||||
|
||||
except FileNotFoundError as e:
|
||||
print(f"❌ Error: {e}")
|
||||
print(f"\nMake sure {yaml_path} exists in the current directory.")
|
||||
print("\nExample agents.yaml structure:")
|
||||
print("""
|
||||
framework: praisonai
|
||||
|
||||
agents:
|
||||
- name: "AI News Monitor"
|
||||
role: "News Analyst"
|
||||
instructions: "Search and summarize AI news"
|
||||
tools:
|
||||
- search_tool
|
||||
|
||||
task: "Search for latest AI news"
|
||||
|
||||
schedule:
|
||||
interval: "hourly"
|
||||
max_retries: 3
|
||||
run_immediately: true
|
||||
""")
|
||||
return 1
|
||||
|
||||
except ValueError as e:
|
||||
print(f"❌ Configuration Error: {e}")
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {e}")
|
||||
if verbose:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def add_schedule_arguments(subparsers):
|
||||
"""
|
||||
Add schedule subcommand and arguments to parser.
|
||||
|
||||
Args:
|
||||
subparsers: Subparsers object from argparse
|
||||
"""
|
||||
schedule_parser = subparsers.add_parser(
|
||||
'schedule',
|
||||
help='Schedule an agent to run continuously at regular intervals'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'schedule_yaml',
|
||||
nargs='?',
|
||||
default='agents.yaml',
|
||||
help='Path to agents.yaml file (default: agents.yaml)'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--interval',
|
||||
dest='schedule_interval',
|
||||
type=str,
|
||||
help='Override schedule interval (e.g., "hourly", "*/30m", "daily")'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--max-retries',
|
||||
dest='schedule_max_retries',
|
||||
type=int,
|
||||
help='Override maximum retry attempts (default: from YAML or 3)'
|
||||
)
|
||||
|
||||
schedule_parser.add_argument(
|
||||
'--verbose', '-v',
|
||||
action='store_true',
|
||||
help='Enable verbose logging'
|
||||
)
|
||||
|
||||
schedule_parser.set_defaults(func=AgentSchedulerHandler.handle_schedule_command)
|
||||
|
||||
|
||||
def setup_scheduler_cli(subparsers):
|
||||
"""
|
||||
Setup scheduler CLI commands.
|
||||
|
||||
Args:
|
||||
subparsers: Subparsers object from argparse
|
||||
"""
|
||||
AgentSchedulerHandler.add_schedule_arguments(subparsers)
|
||||
@@ -0,0 +1,375 @@
|
||||
"""
|
||||
Multi-Agent CLI Feature Handler
|
||||
|
||||
Provides CLI commands for defining and running multiple agents:
|
||||
- praisonai agents run --agent "name:role:tools" --agent "..." --task "..."
|
||||
|
||||
Example:
|
||||
praisonai agents run \
|
||||
--agent "researcher:Research Analyst:internet_search" \
|
||||
--agent "writer:Content Writer:write_file" \
|
||||
--task "Research AI trends and write a report"
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Dict, Any
|
||||
|
||||
|
||||
def parse_agent_definition(definition: str) -> Dict[str, Any]:
|
||||
"""Parse an agent definition string.
|
||||
|
||||
Format: name:role:tools:goal (tools and goal are optional)
|
||||
|
||||
Args:
|
||||
definition: Agent definition string
|
||||
|
||||
Returns:
|
||||
Dictionary with name, role, tools, and optionally goal
|
||||
"""
|
||||
parts = definition.split(':')
|
||||
|
||||
result = {
|
||||
'name': parts[0].strip() if len(parts) > 0 else 'agent',
|
||||
'role': parts[1].strip() if len(parts) > 1 else 'Assistant',
|
||||
'tools': [],
|
||||
'goal': None
|
||||
}
|
||||
|
||||
# Parse tools (comma-separated)
|
||||
if len(parts) > 2 and parts[2].strip():
|
||||
result['tools'] = [t.strip() for t in parts[2].split(',') if t.strip()]
|
||||
|
||||
# Parse goal if provided
|
||||
if len(parts) > 3:
|
||||
result['goal'] = parts[3].strip()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
class MultiAgentHandler:
|
||||
"""Handler for multi-agent CLI commands."""
|
||||
|
||||
def __init__(self, verbose: bool = True):
|
||||
"""Initialize the multi-agent handler.
|
||||
|
||||
Args:
|
||||
verbose: Whether to print verbose output
|
||||
"""
|
||||
self.verbose = verbose
|
||||
|
||||
def create_agents_from_definitions(
|
||||
self,
|
||||
definitions: List[str],
|
||||
llm: Optional[str] = None,
|
||||
instructions: Optional[str] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Create agent configurations from definition strings.
|
||||
|
||||
Args:
|
||||
definitions: List of agent definition strings
|
||||
llm: LLM model to use for all agents
|
||||
instructions: Additional instructions for all agents
|
||||
|
||||
Returns:
|
||||
List of agent configuration dictionaries
|
||||
"""
|
||||
agents = []
|
||||
|
||||
for definition in definitions:
|
||||
agent_config = parse_agent_definition(definition)
|
||||
|
||||
if llm:
|
||||
agent_config['llm'] = llm
|
||||
|
||||
if instructions:
|
||||
agent_config['instructions'] = instructions
|
||||
|
||||
agents.append(agent_config)
|
||||
|
||||
return agents
|
||||
|
||||
def prepare_execution_config(
|
||||
self,
|
||||
agent_definitions: List[str],
|
||||
task: str,
|
||||
process: str = "sequential",
|
||||
llm: Optional[str] = None,
|
||||
instructions: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""Prepare configuration for agent execution.
|
||||
|
||||
Args:
|
||||
agent_definitions: List of agent definition strings
|
||||
task: Task description
|
||||
process: Execution process (sequential, parallel)
|
||||
llm: LLM model to use
|
||||
instructions: Additional instructions
|
||||
|
||||
Returns:
|
||||
Execution configuration dictionary
|
||||
"""
|
||||
agents = self.create_agents_from_definitions(
|
||||
agent_definitions, llm, instructions
|
||||
)
|
||||
|
||||
return {
|
||||
'agents': agents,
|
||||
'task': task,
|
||||
'process': process,
|
||||
'llm': llm
|
||||
}
|
||||
|
||||
def _load_tools(self, tool_names: List[str]) -> List:
|
||||
"""Load tool functions by name.
|
||||
|
||||
Args:
|
||||
tool_names: List of tool names to load
|
||||
|
||||
Returns:
|
||||
List of tool functions
|
||||
"""
|
||||
tools = []
|
||||
|
||||
try:
|
||||
from praisonaiagents.tools import (
|
||||
internet_search, read_file, write_file, list_files,
|
||||
execute_command, read_csv, write_csv, analyze_csv
|
||||
)
|
||||
|
||||
tool_map = {
|
||||
'internet_search': internet_search,
|
||||
'read_file': read_file,
|
||||
'write_file': write_file,
|
||||
'list_files': list_files,
|
||||
'execute_command': execute_command,
|
||||
'read_csv': read_csv,
|
||||
'write_csv': write_csv,
|
||||
'analyze_csv': analyze_csv,
|
||||
}
|
||||
|
||||
for name in tool_names:
|
||||
if name in tool_map:
|
||||
tools.append(tool_map[name])
|
||||
else:
|
||||
if self.verbose:
|
||||
print(f"⚠ Tool '{name}' not found, skipping")
|
||||
except ImportError as e:
|
||||
if self.verbose:
|
||||
print(f"⚠ Could not load tools: {e}")
|
||||
|
||||
return tools
|
||||
|
||||
def _execute_agents(
|
||||
self,
|
||||
agents_config: List[Dict[str, Any]],
|
||||
task: str,
|
||||
process: str = "sequential",
|
||||
llm: Optional[str] = None
|
||||
) -> str:
|
||||
"""Execute agents with the given configuration.
|
||||
|
||||
Args:
|
||||
agents_config: List of agent configurations
|
||||
task: Task to execute
|
||||
process: Execution process
|
||||
llm: LLM model to use
|
||||
|
||||
Returns:
|
||||
Execution result
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents import Agent, Task, PraisonAIAgents
|
||||
|
||||
# Create Agent objects
|
||||
agents = []
|
||||
tasks = []
|
||||
|
||||
for i, config in enumerate(agents_config):
|
||||
# Load tools for this agent
|
||||
tools = self._load_tools(config.get('tools', []))
|
||||
|
||||
agent = Agent(
|
||||
name=config['name'],
|
||||
role=config['role'],
|
||||
goal=config.get('goal') or f"Complete tasks as {config['role']}",
|
||||
backstory=f"You are a skilled {config['role']}.",
|
||||
instructions=config.get('instructions', ''),
|
||||
tools=tools if tools else None,
|
||||
llm=config.get('llm') or llm or os.environ.get('OPENAI_MODEL_NAME', 'gpt-4o-mini'),
|
||||
verbose=self.verbose
|
||||
)
|
||||
agents.append(agent)
|
||||
|
||||
# Create task for this agent
|
||||
agent_task = Task(
|
||||
name=f"task_{config['name']}",
|
||||
description=task if i == 0 else f"Continue the work: {task}",
|
||||
expected_output="Completed task output",
|
||||
agent=agent
|
||||
)
|
||||
tasks.append(agent_task)
|
||||
|
||||
# Run agents
|
||||
if self.verbose:
|
||||
print(f"\n🚀 Running {len(agents)} agents ({process} mode)...")
|
||||
for agent in agents:
|
||||
print(f" - {agent.name}: {agent.role}")
|
||||
print()
|
||||
|
||||
praison_agents = PraisonAIAgents(
|
||||
agents=agents,
|
||||
tasks=tasks,
|
||||
process=process,
|
||||
verbose=self.verbose
|
||||
)
|
||||
|
||||
result = praison_agents.start()
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
if self.verbose:
|
||||
print(f"❌ Execution failed: {e}")
|
||||
raise
|
||||
|
||||
def run(
|
||||
self,
|
||||
agent_definitions: List[str],
|
||||
task: str,
|
||||
process: str = "sequential",
|
||||
llm: Optional[str] = None,
|
||||
instructions: Optional[str] = None
|
||||
) -> str:
|
||||
"""Run multiple agents with the given task.
|
||||
|
||||
Args:
|
||||
agent_definitions: List of agent definition strings
|
||||
task: Task to execute
|
||||
process: Execution process (sequential, parallel)
|
||||
llm: LLM model to use
|
||||
instructions: Additional instructions
|
||||
|
||||
Returns:
|
||||
Execution result
|
||||
"""
|
||||
config = self.prepare_execution_config(
|
||||
agent_definitions, task, process, llm, instructions
|
||||
)
|
||||
|
||||
return self._execute_agents(
|
||||
config['agents'],
|
||||
config['task'],
|
||||
config['process'],
|
||||
config['llm']
|
||||
)
|
||||
|
||||
|
||||
def handle_agents_command(args) -> int:
|
||||
"""Handle agents subcommand from CLI.
|
||||
|
||||
Args:
|
||||
args: Parsed command line arguments
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, non-zero for error)
|
||||
"""
|
||||
handler = MultiAgentHandler(verbose=True)
|
||||
|
||||
try:
|
||||
if args.agents_command == "run":
|
||||
if not hasattr(args, 'agent') or not args.agent:
|
||||
print("Error: At least one --agent is required")
|
||||
return 1
|
||||
|
||||
if not hasattr(args, 'task') or not args.task:
|
||||
print("Error: --task is required")
|
||||
return 1
|
||||
|
||||
result = handler.run(
|
||||
agent_definitions=args.agent,
|
||||
task=args.task,
|
||||
process=getattr(args, 'process', 'sequential'),
|
||||
llm=getattr(args, 'llm', None),
|
||||
instructions=getattr(args, 'instructions', None)
|
||||
)
|
||||
|
||||
print("\n" + "="*50)
|
||||
print("RESULT:")
|
||||
print("="*50)
|
||||
print(result)
|
||||
|
||||
elif args.agents_command == "list":
|
||||
# List available tools
|
||||
print("\nAvailable tools for agents:")
|
||||
print("-" * 30)
|
||||
tools = [
|
||||
("internet_search", "Search the web"),
|
||||
("read_file", "Read file contents"),
|
||||
("write_file", "Write to a file"),
|
||||
("list_files", "List directory contents"),
|
||||
("execute_command", "Execute shell commands"),
|
||||
("read_csv", "Read CSV files"),
|
||||
("write_csv", "Write CSV files"),
|
||||
("analyze_csv", "Analyze CSV data"),
|
||||
]
|
||||
for name, desc in tools:
|
||||
print(f" {name}: {desc}")
|
||||
|
||||
else:
|
||||
print(f"Unknown agents command: {args.agents_command}")
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def add_agents_parser(subparsers) -> None:
|
||||
"""Add agents subcommand to argument parser.
|
||||
|
||||
Args:
|
||||
subparsers: Subparsers object from argparse
|
||||
"""
|
||||
# run command
|
||||
run_parser = subparsers.add_parser(
|
||||
'run',
|
||||
help='Run multiple agents with a task'
|
||||
)
|
||||
run_parser.add_argument(
|
||||
'--agent', '-a',
|
||||
action='append',
|
||||
required=True,
|
||||
help='Agent definition (format: name:role:tools). Can be used multiple times.'
|
||||
)
|
||||
run_parser.add_argument(
|
||||
'--task', '-t',
|
||||
required=True,
|
||||
help='Task for the agents to complete'
|
||||
)
|
||||
run_parser.add_argument(
|
||||
'--process', '-p',
|
||||
choices=['sequential', 'parallel'],
|
||||
default='sequential',
|
||||
help='Execution process (default: sequential)'
|
||||
)
|
||||
run_parser.add_argument(
|
||||
'--llm', '-m',
|
||||
help='LLM model to use for all agents'
|
||||
)
|
||||
run_parser.add_argument(
|
||||
'--instructions', '-i',
|
||||
help='Additional instructions for all agents'
|
||||
)
|
||||
run_parser.add_argument(
|
||||
'--verbose', '-v',
|
||||
action='store_true',
|
||||
help='Enable verbose output'
|
||||
)
|
||||
|
||||
# list command - list available tools
|
||||
list_parser = subparsers.add_parser(
|
||||
'list',
|
||||
help='List available tools for agents'
|
||||
)
|
||||
@@ -0,0 +1,471 @@
|
||||
"""
|
||||
@ Mention Autocomplete for PraisonAI CLI.
|
||||
|
||||
Provides file/directory autocomplete when user types @.
|
||||
Uses only stdlib imports for zero performance impact.
|
||||
|
||||
Features:
|
||||
- Detect @ trigger in input
|
||||
- Fuzzy search files/directories
|
||||
- Cache results for performance
|
||||
- Integrate with prompt_toolkit
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import fnmatch
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Dict, Tuple
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Data Classes
|
||||
# ============================================================================
|
||||
|
||||
@dataclass
|
||||
class AtMentionContext:
|
||||
"""Context for an @ mention in user input."""
|
||||
is_active: bool
|
||||
query: str
|
||||
start_pos: int
|
||||
mention_type: str = "file" # file, directory, web, etc.
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileSuggestion:
|
||||
"""A file or directory suggestion."""
|
||||
path: str
|
||||
file_type: str # "file" or "directory"
|
||||
score: int = 100
|
||||
display: str = ""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.display:
|
||||
self.display = self.path
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# @ Mention Detection
|
||||
# ============================================================================
|
||||
|
||||
def detect_at_mention(text: str, cursor_pos: int) -> Optional[AtMentionContext]:
|
||||
"""
|
||||
Detect if cursor is within an @ mention context.
|
||||
|
||||
Scans backward from cursor to find @ symbol.
|
||||
Returns None if not in @ context.
|
||||
|
||||
Args:
|
||||
text: Full input text
|
||||
cursor_pos: Current cursor position
|
||||
|
||||
Returns:
|
||||
AtMentionContext if in @ mention, None otherwise
|
||||
"""
|
||||
if cursor_pos <= 0 or cursor_pos > len(text):
|
||||
return None
|
||||
|
||||
# Scan backward from cursor to find @
|
||||
at_pos = -1
|
||||
for i in range(cursor_pos - 1, -1, -1):
|
||||
char = text[i]
|
||||
|
||||
# Found @
|
||||
if char == '@':
|
||||
at_pos = i
|
||||
break
|
||||
|
||||
# Space/newline breaks @ context
|
||||
if char in ' \t\n\r':
|
||||
return None
|
||||
|
||||
if at_pos == -1:
|
||||
return None
|
||||
|
||||
# Extract query between @ and cursor
|
||||
query = text[at_pos + 1:cursor_pos]
|
||||
|
||||
return AtMentionContext(
|
||||
is_active=True,
|
||||
query=query,
|
||||
start_pos=at_pos
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# File Search Service
|
||||
# ============================================================================
|
||||
|
||||
class FileSearchService:
|
||||
"""
|
||||
Service for searching files in a directory.
|
||||
|
||||
Features:
|
||||
- Walk directory tree
|
||||
- Fuzzy match file paths
|
||||
- Cache results with TTL
|
||||
- Respect .gitignore (basic)
|
||||
- Support absolute paths (@/Users/...) and home paths (@~/...)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
root_dir: str,
|
||||
cache_ttl: int = 30,
|
||||
max_depth: int = 5,
|
||||
ignore_patterns: Optional[List[str]] = None
|
||||
):
|
||||
self._root = os.path.abspath(root_dir)
|
||||
self._cache_ttl = cache_ttl
|
||||
self._max_depth = max_depth
|
||||
self._ignore_patterns = ignore_patterns or [
|
||||
'.git', '__pycache__', 'node_modules', '.venv', 'venv',
|
||||
'*.pyc', '*.pyo', '.DS_Store', '*.egg-info'
|
||||
]
|
||||
|
||||
# Cache: query -> (results, timestamp)
|
||||
self._cache: Dict[str, Tuple[List[FileSuggestion], float]] = {}
|
||||
|
||||
# File list cache per directory
|
||||
self._file_list: Optional[List[str]] = None
|
||||
self._file_list_time: float = 0
|
||||
self._file_list_dir: Optional[str] = None
|
||||
|
||||
def _should_ignore(self, name: str) -> bool:
|
||||
"""Check if file/dir should be ignored."""
|
||||
for pattern in self._ignore_patterns:
|
||||
if fnmatch.fnmatch(name, pattern):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _get_file_list(self, search_dir: str = None) -> List[str]:
|
||||
"""Get list of all files (cached)."""
|
||||
now = time.time()
|
||||
target_dir = search_dir or self._root
|
||||
|
||||
# Return cached if fresh and same directory
|
||||
if (self._file_list is not None and
|
||||
self._file_list_dir == target_dir and
|
||||
(now - self._file_list_time) < self._cache_ttl):
|
||||
return self._file_list
|
||||
|
||||
files = []
|
||||
|
||||
try:
|
||||
for root, dirs, filenames in os.walk(target_dir):
|
||||
# Calculate depth
|
||||
rel_root = os.path.relpath(root, target_dir)
|
||||
depth = 0 if rel_root == '.' else rel_root.count(os.sep) + 1
|
||||
|
||||
if depth > self._max_depth:
|
||||
dirs.clear() # Don't descend further
|
||||
continue
|
||||
|
||||
# Filter ignored directories
|
||||
dirs[:] = [d for d in dirs if not self._should_ignore(d)]
|
||||
|
||||
# Add directories
|
||||
for d in dirs:
|
||||
rel_path = os.path.relpath(os.path.join(root, d), target_dir)
|
||||
files.append(rel_path + os.sep)
|
||||
|
||||
# Add files
|
||||
for f in filenames:
|
||||
if not self._should_ignore(f):
|
||||
rel_path = os.path.relpath(os.path.join(root, f), target_dir)
|
||||
files.append(rel_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
self._file_list = files
|
||||
self._file_list_time = now
|
||||
self._file_list_dir = target_dir
|
||||
return files
|
||||
|
||||
def _fuzzy_match(self, text: str, query: str) -> int:
|
||||
"""
|
||||
Simple fuzzy match scoring.
|
||||
|
||||
Returns score 0-100, higher is better.
|
||||
"""
|
||||
if not query:
|
||||
return 50 # Default score for empty query
|
||||
|
||||
text_lower = text.lower()
|
||||
query_lower = query.lower()
|
||||
|
||||
# Exact match
|
||||
if text_lower == query_lower:
|
||||
return 100
|
||||
|
||||
# Starts with
|
||||
if text_lower.startswith(query_lower):
|
||||
return 90
|
||||
|
||||
# Contains
|
||||
if query_lower in text_lower:
|
||||
return 80
|
||||
|
||||
# Fuzzy: all query chars appear in order
|
||||
text_idx = 0
|
||||
query_idx = 0
|
||||
matches = 0
|
||||
|
||||
while text_idx < len(text_lower) and query_idx < len(query_lower):
|
||||
if text_lower[text_idx] == query_lower[query_idx]:
|
||||
matches += 1
|
||||
query_idx += 1
|
||||
text_idx += 1
|
||||
|
||||
if query_idx == len(query_lower):
|
||||
# All query chars found
|
||||
return 50 + int(30 * matches / len(text))
|
||||
|
||||
return 0
|
||||
|
||||
def search(self, query: str, max_results: int = 20) -> List[FileSuggestion]:
|
||||
"""
|
||||
Search for files matching query.
|
||||
|
||||
Supports:
|
||||
- Relative paths: @file.txt, @src/main.py
|
||||
- Home paths: @~/Documents, @~/.config
|
||||
- Absolute paths: @/Users/name/file.txt, @/etc/hosts
|
||||
|
||||
Args:
|
||||
query: Search query (fuzzy matched)
|
||||
max_results: Maximum results to return
|
||||
|
||||
Returns:
|
||||
List of FileSuggestion sorted by score
|
||||
"""
|
||||
# Determine search directory based on query
|
||||
search_dir = self._root
|
||||
search_query = query
|
||||
prefix = ""
|
||||
|
||||
# Handle home directory paths (@~/)
|
||||
if query.startswith('~'):
|
||||
expanded = os.path.expanduser(query)
|
||||
# Find the directory part and filename part
|
||||
if os.path.isdir(expanded):
|
||||
search_dir = expanded
|
||||
search_query = ""
|
||||
prefix = query.rstrip('/') + '/'
|
||||
else:
|
||||
parent = os.path.dirname(expanded)
|
||||
if os.path.isdir(parent):
|
||||
search_dir = parent
|
||||
search_query = os.path.basename(expanded)
|
||||
prefix = os.path.dirname(query).rstrip('/') + '/'
|
||||
else:
|
||||
# Just expand ~ and search from home
|
||||
search_dir = os.path.expanduser('~')
|
||||
search_query = query[2:] if query.startswith('~/') else query[1:]
|
||||
prefix = "~/"
|
||||
|
||||
# Handle absolute paths (@/Users/..., @/etc/...)
|
||||
elif query.startswith('/'):
|
||||
if os.path.isdir(query):
|
||||
search_dir = query
|
||||
search_query = ""
|
||||
prefix = query.rstrip('/') + '/'
|
||||
else:
|
||||
parent = os.path.dirname(query)
|
||||
if os.path.isdir(parent):
|
||||
search_dir = parent
|
||||
search_query = os.path.basename(query)
|
||||
prefix = parent.rstrip('/') + '/'
|
||||
else:
|
||||
# Search from root with the query
|
||||
search_dir = '/'
|
||||
search_query = query[1:]
|
||||
prefix = "/"
|
||||
|
||||
# Check cache
|
||||
cache_key = f"{search_dir}:{search_query}:{max_results}"
|
||||
now = time.time()
|
||||
|
||||
if cache_key in self._cache:
|
||||
results, cached_time = self._cache[cache_key]
|
||||
if (now - cached_time) < self._cache_ttl:
|
||||
return results
|
||||
|
||||
# Get all files from the appropriate directory
|
||||
all_files = self._get_file_list(search_dir)
|
||||
|
||||
# Score and filter
|
||||
scored: List[Tuple[int, str]] = []
|
||||
for path in all_files:
|
||||
score = self._fuzzy_match(path, search_query)
|
||||
if score > 0:
|
||||
scored.append((score, path))
|
||||
|
||||
# Sort by score descending
|
||||
scored.sort(key=lambda x: (-x[0], x[1]))
|
||||
|
||||
# Build results with proper prefix
|
||||
results = []
|
||||
for score, path in scored[:max_results]:
|
||||
file_type = "directory" if path.endswith(os.sep) else "file"
|
||||
display_path = prefix + path if prefix else path
|
||||
results.append(FileSuggestion(
|
||||
path=display_path,
|
||||
file_type=file_type,
|
||||
score=score
|
||||
))
|
||||
|
||||
# Cache results
|
||||
self._cache[cache_key] = (results, now)
|
||||
|
||||
return results
|
||||
|
||||
def clear_cache(self) -> None:
|
||||
"""Clear all caches."""
|
||||
self._cache.clear()
|
||||
self._file_list = None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Completers for prompt_toolkit
|
||||
# ============================================================================
|
||||
|
||||
# Lazy import to avoid performance impact
|
||||
_Completer = None
|
||||
_Completion = None
|
||||
|
||||
def _get_prompt_toolkit():
|
||||
"""Lazy import prompt_toolkit."""
|
||||
global _Completer, _Completion
|
||||
if _Completer is None:
|
||||
from prompt_toolkit.completion import Completer, Completion
|
||||
_Completer = Completer
|
||||
_Completion = Completion
|
||||
return _Completer, _Completion
|
||||
|
||||
|
||||
class AtMentionCompleter:
|
||||
"""
|
||||
Completer for @ mentions.
|
||||
|
||||
Shows file suggestions when user types @.
|
||||
Implements prompt_toolkit Completer interface.
|
||||
"""
|
||||
|
||||
def __init__(self, root_dir: str):
|
||||
self._file_service = FileSearchService(root_dir)
|
||||
|
||||
def get_completions(self, document, complete_event):
|
||||
"""Get completions for @ mentions."""
|
||||
_, Completion = _get_prompt_toolkit()
|
||||
|
||||
text = document.text_before_cursor
|
||||
cursor_pos = len(text)
|
||||
|
||||
# Detect @ context
|
||||
context = detect_at_mention(text, cursor_pos)
|
||||
if context is None or not context.is_active:
|
||||
return
|
||||
|
||||
# Search files
|
||||
results = self._file_service.search(context.query, max_results=15)
|
||||
|
||||
# Yield completions
|
||||
for suggestion in results:
|
||||
# Icon based on type
|
||||
icon = "📁 " if suggestion.file_type == "directory" else "📄 "
|
||||
display = f"{icon}{suggestion.path}"
|
||||
|
||||
# Calculate replacement position
|
||||
# Replace from @ to cursor
|
||||
start_position = -(cursor_pos - context.start_pos)
|
||||
|
||||
yield Completion(
|
||||
text=f"@{suggestion.path}",
|
||||
start_position=start_position,
|
||||
display=display
|
||||
)
|
||||
|
||||
|
||||
def create_combined_completer(commands: List[str], root_dir: str):
|
||||
"""
|
||||
Factory function to create a CombinedCompleter.
|
||||
|
||||
Returns a proper Completer subclass that handles both / and @.
|
||||
"""
|
||||
Completer, Completion = _get_prompt_toolkit()
|
||||
file_service = FileSearchService(root_dir)
|
||||
|
||||
class CombinedCompleter(Completer):
|
||||
"""
|
||||
Combined completer for both / commands and @ mentions.
|
||||
|
||||
Properly inherits from prompt_toolkit Completer.
|
||||
"""
|
||||
|
||||
def get_completions(self, document, complete_event):
|
||||
"""Get completions for / or @."""
|
||||
text = document.text_before_cursor.lstrip()
|
||||
|
||||
# Check for slash command
|
||||
if text.startswith('/'):
|
||||
cmd_text = text[1:].lower()
|
||||
for cmd in commands:
|
||||
if cmd.lower().startswith(cmd_text):
|
||||
yield Completion(
|
||||
f'/{cmd}',
|
||||
start_position=-len(text),
|
||||
display=f'/{cmd}'
|
||||
)
|
||||
return
|
||||
|
||||
# Check for @ mention
|
||||
full_text = document.text_before_cursor
|
||||
cursor_pos = len(full_text)
|
||||
|
||||
context = detect_at_mention(full_text, cursor_pos)
|
||||
if context is None or not context.is_active:
|
||||
return
|
||||
|
||||
# Search files
|
||||
results = file_service.search(context.query, max_results=15)
|
||||
|
||||
# Yield completions
|
||||
for suggestion in results:
|
||||
icon = "📁 " if suggestion.file_type == "directory" else "📄 "
|
||||
display = f"{icon}{suggestion.path}"
|
||||
start_position = -(cursor_pos - context.start_pos)
|
||||
|
||||
yield Completion(
|
||||
text=f"@{suggestion.path}",
|
||||
start_position=start_position,
|
||||
display=display
|
||||
)
|
||||
|
||||
return CombinedCompleter()
|
||||
|
||||
|
||||
# Backward compatible alias
|
||||
class CombinedCompleter:
|
||||
"""
|
||||
Wrapper class for backward compatibility.
|
||||
|
||||
Use create_combined_completer() for proper Completer subclass.
|
||||
"""
|
||||
|
||||
def __new__(cls, commands: List[str], root_dir: str):
|
||||
return create_combined_completer(commands, root_dir)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Exports
|
||||
# ============================================================================
|
||||
|
||||
__all__ = [
|
||||
'AtMentionContext',
|
||||
'FileSuggestion',
|
||||
'detect_at_mention',
|
||||
'FileSearchService',
|
||||
'AtMentionCompleter',
|
||||
'CombinedCompleter',
|
||||
]
|
||||
@@ -45,8 +45,10 @@ class AutoMemoryHandler(FlagHandler):
|
||||
def _get_auto_memory(self, user_id: str = None):
|
||||
"""Get AutoMemory instance lazily."""
|
||||
try:
|
||||
from praisonaiagents.memory import AutoMemory
|
||||
return AutoMemory(user_id=user_id or "default")
|
||||
from praisonaiagents.memory import AutoMemory, FileMemory
|
||||
# AutoMemory requires a FileMemory instance
|
||||
memory = FileMemory(user_id=user_id or "default")
|
||||
return AutoMemory(memory=memory)
|
||||
except ImportError:
|
||||
self.print_status(
|
||||
"AutoMemory requires praisonaiagents. Install with: pip install praisonaiagents",
|
||||
@@ -71,33 +73,36 @@ class AutoMemoryHandler(FlagHandler):
|
||||
config['auto_memory_user_id'] = flag_value.get('user_id', 'default')
|
||||
return config
|
||||
|
||||
def extract_memories(self, text: str, user_id: str = None) -> Dict[str, Any]:
|
||||
def extract_memories(self, text: str, user_id: str = None, user_message: str = None) -> list:
|
||||
"""
|
||||
Extract memories from text.
|
||||
Extract and store memories from text.
|
||||
|
||||
Args:
|
||||
text: Text to extract memories from
|
||||
text: Assistant response text to extract memories from
|
||||
user_id: User ID for memory isolation
|
||||
user_message: Original user message (for context)
|
||||
|
||||
Returns:
|
||||
Dictionary of extracted memories
|
||||
List of extracted memories
|
||||
"""
|
||||
auto_memory = self._get_auto_memory(user_id)
|
||||
if not auto_memory:
|
||||
return {}
|
||||
return []
|
||||
|
||||
try:
|
||||
if hasattr(auto_memory, 'extract'):
|
||||
memories = auto_memory.extract(text)
|
||||
self.print_status(f"🧠 Extracted {len(memories)} memories", "success")
|
||||
return memories
|
||||
elif hasattr(auto_memory, 'process'):
|
||||
result = auto_memory.process(text)
|
||||
return result
|
||||
# Use process_interaction which extracts AND stores memories
|
||||
memories = auto_memory.process_interaction(
|
||||
user_message=user_message or text,
|
||||
assistant_response=text if user_message else None,
|
||||
store=True # Store the memories
|
||||
)
|
||||
if memories:
|
||||
self.print_status(f"🧠 Extracted and stored {len(memories)} memories", "success")
|
||||
return memories
|
||||
except Exception as e:
|
||||
self.log(f"Memory extraction failed: {e}", "error")
|
||||
|
||||
return {}
|
||||
return []
|
||||
|
||||
def store_memory(self, content: str, user_id: str = None, importance: float = 0.5) -> bool:
|
||||
"""
|
||||
@@ -127,13 +132,14 @@ class AutoMemoryHandler(FlagHandler):
|
||||
self.log(f"Memory storage failed: {e}", "error")
|
||||
return False
|
||||
|
||||
def post_process_result(self, result: Any, flag_value: Any) -> Any:
|
||||
def post_process_result(self, result: Any, flag_value: Any, user_message: str = None) -> Any:
|
||||
"""
|
||||
Post-process result to extract and store memories.
|
||||
|
||||
Args:
|
||||
result: Agent output
|
||||
flag_value: Boolean or dict with configuration
|
||||
user_message: Original user message for context
|
||||
|
||||
Returns:
|
||||
Original result (memories are stored)
|
||||
@@ -144,15 +150,18 @@ class AutoMemoryHandler(FlagHandler):
|
||||
user_id = None
|
||||
if isinstance(flag_value, dict):
|
||||
user_id = flag_value.get('user_id')
|
||||
user_message = flag_value.get('user_message', user_message)
|
||||
|
||||
# Extract memories from result
|
||||
# Extract and store memories from result
|
||||
text = str(result)
|
||||
memories = self.extract_memories(text, user_id)
|
||||
memories = self.extract_memories(text, user_id, user_message=user_message)
|
||||
|
||||
if memories:
|
||||
self.print_status("\n🧠 Auto-extracted Memories:", "info")
|
||||
for key, value in memories.items():
|
||||
self.print_status(f" • {key}: {value[:50]}...", "info")
|
||||
for mem in memories:
|
||||
mem_type = mem.get('type', 'unknown')
|
||||
content = mem.get('content', str(mem))[:50]
|
||||
self.print_status(f" • {mem_type}: {content}...", "info")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@@ -0,0 +1,534 @@
|
||||
"""
|
||||
CLI Compare Feature for PraisonAI.
|
||||
|
||||
Compares different CLI command modes without performance impact.
|
||||
Only loaded when --compare flag is used.
|
||||
"""
|
||||
|
||||
import time
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, Any, List, Optional
|
||||
from datetime import datetime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
COMPARE_MODES = {
|
||||
"basic": {},
|
||||
"tools": {"tools": "calculator,internet_search"},
|
||||
"research": {"research": True},
|
||||
"planning": {"planning": True},
|
||||
"memory": {"memory": True},
|
||||
"router": {"router": True},
|
||||
"web_search": {"web_search": True},
|
||||
"web_fetch": {"web_fetch": True},
|
||||
"query_rewrite": {"query_rewrite": True},
|
||||
"expand_prompt": {"expand_prompt": True},
|
||||
}
|
||||
|
||||
|
||||
def get_mode_config(mode_name: str, custom_args: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Get configuration for a comparison mode.
|
||||
|
||||
Args:
|
||||
mode_name: Name of the mode
|
||||
custom_args: Optional custom arguments to merge
|
||||
|
||||
Returns:
|
||||
Configuration dictionary for the mode
|
||||
"""
|
||||
config = COMPARE_MODES.get(mode_name, {}).copy()
|
||||
if custom_args:
|
||||
config.update(custom_args)
|
||||
return config
|
||||
|
||||
|
||||
def list_available_modes() -> List[str]:
|
||||
"""
|
||||
List all available comparison modes.
|
||||
|
||||
Returns:
|
||||
List of mode names
|
||||
"""
|
||||
return list(COMPARE_MODES.keys())
|
||||
|
||||
|
||||
def parse_modes(modes_str: str) -> List[str]:
|
||||
"""
|
||||
Parse comma-separated modes string.
|
||||
|
||||
Args:
|
||||
modes_str: Comma-separated mode names
|
||||
|
||||
Returns:
|
||||
List of mode names
|
||||
"""
|
||||
if not modes_str or not modes_str.strip():
|
||||
return ["basic"]
|
||||
|
||||
modes = [m.strip() for m in modes_str.split(",")]
|
||||
return [m for m in modes if m]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModeResult:
|
||||
"""Result from running a single mode."""
|
||||
mode: str
|
||||
output: str
|
||||
execution_time_ms: float
|
||||
model_used: str
|
||||
tokens: Optional[Dict[str, int]] = None
|
||||
cost: Optional[float] = None
|
||||
tools_used: List[str] = field(default_factory=list)
|
||||
error: Optional[str] = None
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dictionary."""
|
||||
return {
|
||||
"mode": self.mode,
|
||||
"output_preview": self.output[:200] + "..." if len(self.output) > 200 else self.output,
|
||||
"execution_time_ms": self.execution_time_ms,
|
||||
"model_used": self.model_used,
|
||||
"tokens": self.tokens,
|
||||
"cost": self.cost,
|
||||
"tools_used": self.tools_used,
|
||||
"error": self.error,
|
||||
"timestamp": self.timestamp,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompareResult:
|
||||
"""Result from comparing multiple modes."""
|
||||
query: str
|
||||
comparisons: List[ModeResult] = field(default_factory=list)
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
|
||||
def get_summary(self) -> Dict[str, Any]:
|
||||
"""Generate summary statistics."""
|
||||
if not self.comparisons:
|
||||
return {}
|
||||
|
||||
times = [(c.mode, c.execution_time_ms) for c in self.comparisons if c.error is None]
|
||||
if not times:
|
||||
return {"error": "All modes failed"}
|
||||
|
||||
sorted_by_time = sorted(times, key=lambda x: x[1])
|
||||
|
||||
summary = {
|
||||
"fastest": sorted_by_time[0][0],
|
||||
"slowest": sorted_by_time[-1][0],
|
||||
"fastest_time_ms": sorted_by_time[0][1],
|
||||
"slowest_time_ms": sorted_by_time[-1][1],
|
||||
}
|
||||
|
||||
total_tokens = 0
|
||||
total_cost = 0.0
|
||||
for c in self.comparisons:
|
||||
if c.tokens:
|
||||
total_tokens += c.tokens.get("input", 0) + c.tokens.get("output", 0)
|
||||
if c.cost:
|
||||
total_cost += c.cost
|
||||
|
||||
if total_tokens > 0:
|
||||
summary["total_tokens"] = total_tokens
|
||||
if total_cost > 0:
|
||||
summary["total_cost"] = total_cost
|
||||
|
||||
return summary
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dictionary."""
|
||||
return {
|
||||
"query": self.query,
|
||||
"timestamp": self.timestamp,
|
||||
"comparisons": [c.to_dict() for c in self.comparisons],
|
||||
"summary": self.get_summary(),
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""Convert to JSON string."""
|
||||
return json.dumps(self.to_dict(), indent=2)
|
||||
|
||||
|
||||
def save_compare_result(result: CompareResult, path: str) -> bool:
|
||||
"""
|
||||
Save comparison result to file.
|
||||
|
||||
Args:
|
||||
result: CompareResult to save
|
||||
path: File path
|
||||
|
||||
Returns:
|
||||
True if successful
|
||||
"""
|
||||
try:
|
||||
from pathlib import Path
|
||||
|
||||
file_path = Path(path)
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_path.write_text(result.to_json())
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to save comparison result: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def format_comparison_table(result: CompareResult, show_responses: bool = True) -> str:
|
||||
"""
|
||||
Format comparison result as a table string with optional response display.
|
||||
|
||||
Args:
|
||||
result: CompareResult to format
|
||||
show_responses: Whether to show actual responses from each mode
|
||||
|
||||
Returns:
|
||||
Formatted table string
|
||||
"""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
from rich.panel import Panel
|
||||
from io import StringIO
|
||||
|
||||
console = Console(file=StringIO(), force_terminal=True)
|
||||
|
||||
# Main comparison table
|
||||
table = Table(title=f"Comparison: {result.query[:50]}...")
|
||||
table.add_column("Mode", style="cyan")
|
||||
table.add_column("Time (ms)", style="green")
|
||||
table.add_column("Model", style="yellow")
|
||||
table.add_column("Tools", style="magenta")
|
||||
table.add_column("Status", style="bold")
|
||||
|
||||
for c in result.comparisons:
|
||||
status = "✅" if c.error is None else f"❌ {c.error[:20]}"
|
||||
tools = ", ".join(c.tools_used) if c.tools_used else "-"
|
||||
table.add_row(
|
||||
c.mode,
|
||||
f"{c.execution_time_ms:.1f}",
|
||||
c.model_used,
|
||||
tools,
|
||||
status
|
||||
)
|
||||
|
||||
summary = result.get_summary()
|
||||
if summary.get("fastest"):
|
||||
table.add_section()
|
||||
table.add_row(
|
||||
"Summary",
|
||||
f"Fastest: {summary['fastest']}",
|
||||
"",
|
||||
"",
|
||||
f"Δ {summary['slowest_time_ms'] - summary['fastest_time_ms']:.1f}ms"
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
|
||||
# Show responses if enabled
|
||||
if show_responses:
|
||||
console.print("\n[bold cyan]📝 Responses:[/bold cyan]")
|
||||
for c in result.comparisons:
|
||||
if c.error is None:
|
||||
# Truncate long responses
|
||||
response_preview = c.output[:500] + "..." if len(c.output) > 500 else c.output
|
||||
console.print(Panel(
|
||||
response_preview,
|
||||
title=f"[bold]{c.mode}[/bold] ({c.execution_time_ms:.0f}ms)",
|
||||
border_style="cyan"
|
||||
))
|
||||
else:
|
||||
console.print(Panel(
|
||||
f"[red]Error: {c.error}[/red]",
|
||||
title=f"[bold]{c.mode}[/bold]",
|
||||
border_style="red"
|
||||
))
|
||||
|
||||
return console.file.getvalue()
|
||||
except ImportError:
|
||||
lines = [f"Comparison: {result.query}"]
|
||||
lines.append("-" * 60)
|
||||
for c in result.comparisons:
|
||||
status = "OK" if c.error is None else f"ERROR: {c.error}"
|
||||
lines.append(f"{c.mode}: {c.execution_time_ms:.1f}ms - {status}")
|
||||
if show_responses and c.error is None:
|
||||
lines.append(f" Response: {c.output[:200]}...")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class CompareHandler:
|
||||
"""
|
||||
Handler for CLI compare feature.
|
||||
|
||||
Compares different CLI command modes and generates comparison reports.
|
||||
"""
|
||||
|
||||
def __init__(self, verbose: bool = False):
|
||||
"""
|
||||
Initialize the compare handler.
|
||||
|
||||
Args:
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
self.verbose = verbose
|
||||
self.results: List[ModeResult] = []
|
||||
|
||||
def compare(
|
||||
self,
|
||||
query: str,
|
||||
modes: List[str],
|
||||
model: Optional[str] = None,
|
||||
**kwargs
|
||||
) -> CompareResult:
|
||||
"""
|
||||
Compare query execution across different modes.
|
||||
|
||||
Args:
|
||||
query: The query to execute
|
||||
modes: List of mode names to compare
|
||||
model: Optional model override
|
||||
**kwargs: Additional arguments
|
||||
|
||||
Returns:
|
||||
CompareResult with all comparisons
|
||||
"""
|
||||
result = CompareResult(query=query)
|
||||
|
||||
for mode in modes:
|
||||
if self.verbose:
|
||||
logger.info(f"Running mode: {mode}")
|
||||
|
||||
try:
|
||||
mode_result = self._run_mode(query, mode, model=model, **kwargs)
|
||||
result.comparisons.append(mode_result)
|
||||
except Exception as e:
|
||||
logger.error(f"Error running mode {mode}: {e}")
|
||||
result.comparisons.append(ModeResult(
|
||||
mode=mode,
|
||||
output="",
|
||||
execution_time_ms=0,
|
||||
model_used=model or "unknown",
|
||||
error=str(e)
|
||||
))
|
||||
|
||||
return result
|
||||
|
||||
def _run_mode(
|
||||
self,
|
||||
query: str,
|
||||
mode: str,
|
||||
model: Optional[str] = None,
|
||||
**kwargs
|
||||
) -> ModeResult:
|
||||
"""
|
||||
Run a single mode and capture results.
|
||||
|
||||
Args:
|
||||
query: The query to execute
|
||||
mode: Mode name
|
||||
model: Optional model override
|
||||
**kwargs: Additional arguments
|
||||
|
||||
Returns:
|
||||
ModeResult with execution details
|
||||
"""
|
||||
config = get_mode_config(mode, kwargs)
|
||||
|
||||
if model:
|
||||
config["llm"] = model
|
||||
|
||||
start_time = time.perf_counter()
|
||||
output = ""
|
||||
tokens = None
|
||||
cost = None
|
||||
tools_used = []
|
||||
error = None
|
||||
model_used = model or "gpt-4o-mini"
|
||||
|
||||
try:
|
||||
from praisonaiagents import Agent
|
||||
|
||||
agent_config = {
|
||||
"name": f"CompareAgent_{mode}",
|
||||
"role": "Assistant",
|
||||
"goal": "Complete the given task",
|
||||
"backstory": "You are a helpful AI assistant",
|
||||
"verbose": False,
|
||||
}
|
||||
|
||||
if config.get("llm"):
|
||||
agent_config["llm"] = config["llm"]
|
||||
model_used = config["llm"]
|
||||
|
||||
if config.get("tools"):
|
||||
tools_list = self._load_tools(config["tools"])
|
||||
if tools_list:
|
||||
agent_config["tools"] = tools_list
|
||||
|
||||
if config.get("planning"):
|
||||
agent_config["planning"] = True
|
||||
|
||||
if config.get("memory"):
|
||||
agent_config["memory"] = True
|
||||
|
||||
if config.get("web_search"):
|
||||
agent_config["web_search"] = True
|
||||
|
||||
if config.get("web_fetch"):
|
||||
agent_config["web_fetch"] = True
|
||||
|
||||
agent = Agent(**agent_config)
|
||||
result = agent.start(query)
|
||||
|
||||
if hasattr(result, 'raw'):
|
||||
output = str(result.raw)
|
||||
else:
|
||||
output = str(result)
|
||||
|
||||
if hasattr(agent, 'tools') and agent.tools:
|
||||
for tool in agent.tools:
|
||||
if hasattr(tool, '__name__'):
|
||||
tools_used.append(tool.__name__)
|
||||
elif hasattr(tool, 'name'):
|
||||
tools_used.append(tool.name)
|
||||
|
||||
except Exception as e:
|
||||
error = str(e)
|
||||
logger.error(f"Mode {mode} failed: {e}")
|
||||
|
||||
end_time = time.perf_counter()
|
||||
execution_time_ms = (end_time - start_time) * 1000
|
||||
|
||||
return ModeResult(
|
||||
mode=mode,
|
||||
output=output,
|
||||
execution_time_ms=execution_time_ms,
|
||||
model_used=model_used,
|
||||
tokens=tokens,
|
||||
cost=cost,
|
||||
tools_used=tools_used,
|
||||
error=error
|
||||
)
|
||||
|
||||
def _load_tools(self, tools_str: str) -> List:
|
||||
"""
|
||||
Load tools from string specification.
|
||||
|
||||
Args:
|
||||
tools_str: Comma-separated tool names
|
||||
|
||||
Returns:
|
||||
List of tool functions
|
||||
"""
|
||||
tools = []
|
||||
tool_names = [t.strip() for t in tools_str.split(",")]
|
||||
|
||||
for name in tool_names:
|
||||
try:
|
||||
if name == "internet_search":
|
||||
from praisonaiagents.tools import internet_search
|
||||
tools.append(internet_search)
|
||||
elif name == "calculator":
|
||||
tools.append(_calculator_tool)
|
||||
except ImportError:
|
||||
logger.warning(f"Could not load tool: {name}")
|
||||
|
||||
return tools
|
||||
|
||||
def print_result(self, result: CompareResult) -> None:
|
||||
"""
|
||||
Print comparison result to console.
|
||||
|
||||
Args:
|
||||
result: CompareResult to print
|
||||
"""
|
||||
table_str = format_comparison_table(result)
|
||||
print(table_str)
|
||||
|
||||
def execute(
|
||||
self,
|
||||
query: str,
|
||||
modes_str: str,
|
||||
model: Optional[str] = None,
|
||||
output_path: Optional[str] = None,
|
||||
**kwargs
|
||||
) -> CompareResult:
|
||||
"""
|
||||
Execute comparison from CLI arguments.
|
||||
|
||||
Args:
|
||||
query: The query to execute
|
||||
modes_str: Comma-separated mode names
|
||||
model: Optional model override
|
||||
output_path: Optional path to save results
|
||||
**kwargs: Additional arguments
|
||||
|
||||
Returns:
|
||||
CompareResult with all comparisons
|
||||
"""
|
||||
modes = parse_modes(modes_str)
|
||||
|
||||
if self.verbose:
|
||||
print(f"[bold cyan]Comparing modes: {', '.join(modes)}[/bold cyan]")
|
||||
|
||||
result = self.compare(query, modes, model=model, **kwargs)
|
||||
|
||||
self.print_result(result)
|
||||
|
||||
if output_path:
|
||||
if save_compare_result(result, output_path):
|
||||
print(f"[green]Results saved to: {output_path}[/green]")
|
||||
else:
|
||||
print(f"[red]Failed to save results to: {output_path}[/red]")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _calculator_tool(expression: str) -> str:
|
||||
"""
|
||||
Evaluate a mathematical expression safely.
|
||||
|
||||
Args:
|
||||
expression: Mathematical expression to evaluate (e.g., "2+2", "15*3", "100/4")
|
||||
|
||||
Returns:
|
||||
The result of the calculation as a string
|
||||
"""
|
||||
import ast
|
||||
import operator
|
||||
|
||||
# Safe operators
|
||||
operators = {
|
||||
ast.Add: operator.add,
|
||||
ast.Sub: operator.sub,
|
||||
ast.Mult: operator.mul,
|
||||
ast.Div: operator.truediv,
|
||||
ast.Pow: operator.pow,
|
||||
ast.USub: operator.neg,
|
||||
ast.UAdd: operator.pos,
|
||||
ast.Mod: operator.mod,
|
||||
ast.FloorDiv: operator.floordiv,
|
||||
}
|
||||
|
||||
def _eval(node):
|
||||
if isinstance(node, ast.Constant):
|
||||
return node.value
|
||||
elif isinstance(node, ast.BinOp):
|
||||
left = _eval(node.left)
|
||||
right = _eval(node.right)
|
||||
return operators[type(node.op)](left, right)
|
||||
elif isinstance(node, ast.UnaryOp):
|
||||
operand = _eval(node.operand)
|
||||
return operators[type(node.op)](operand)
|
||||
else:
|
||||
raise ValueError(f"Unsupported operation: {type(node)}")
|
||||
|
||||
try:
|
||||
tree = ast.parse(expression, mode='eval')
|
||||
result = _eval(tree.body)
|
||||
return str(result)
|
||||
except Exception as e:
|
||||
return f"Error evaluating '{expression}': {e}"
|
||||
@@ -0,0 +1,560 @@
|
||||
"""
|
||||
Evaluation CLI feature for PraisonAI.
|
||||
|
||||
Provides CLI commands for running agent evaluations.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
from typing import Optional, List, Dict, Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EvalHandler:
|
||||
"""Handler for evaluation CLI commands."""
|
||||
|
||||
def __init__(self, verbose: bool = False):
|
||||
"""
|
||||
Initialize the evaluation handler.
|
||||
|
||||
Args:
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
self.verbose = verbose
|
||||
|
||||
def run_accuracy(
|
||||
self,
|
||||
agent_file: Optional[str] = None,
|
||||
input_text: str = "",
|
||||
expected_output: str = "",
|
||||
iterations: int = 1,
|
||||
model: Optional[str] = None,
|
||||
output_file: Optional[str] = None,
|
||||
prompt: Optional[str] = None,
|
||||
llm: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Run accuracy evaluation on an agent.
|
||||
|
||||
Args:
|
||||
agent_file: Path to agents.yaml file (optional if prompt is provided)
|
||||
input_text: Input to provide to the agent
|
||||
expected_output: Expected output to compare against
|
||||
iterations: Number of evaluation iterations
|
||||
model: LLM model for judging
|
||||
output_file: Path to save results
|
||||
prompt: Direct prompt (alternative to agent_file)
|
||||
llm: LLM model for the agent (when using prompt)
|
||||
|
||||
Returns:
|
||||
Evaluation result dictionary
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents.eval import AccuracyEvaluator
|
||||
from praisonaiagents import Agent
|
||||
except ImportError as e:
|
||||
logger.error(f"Failed to import evaluation modules: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
try:
|
||||
# Create agent either from file or from prompt
|
||||
if prompt:
|
||||
# Direct prompt mode - create agent on the fly
|
||||
agent = Agent(
|
||||
name="EvalAgent",
|
||||
role="Assistant",
|
||||
goal="Complete the given task",
|
||||
backstory="You are a helpful assistant.",
|
||||
llm=llm or model or "gpt-4o-mini",
|
||||
verbose=False
|
||||
)
|
||||
# Use prompt as input if input_text not provided
|
||||
if not input_text:
|
||||
input_text = prompt
|
||||
elif agent_file:
|
||||
# Load from agents.yaml
|
||||
try:
|
||||
from praisonai.agents_generator import AgentsGenerator
|
||||
generator = AgentsGenerator(agent_file)
|
||||
agents = generator.generate_agents()
|
||||
|
||||
if not agents:
|
||||
return {"error": "No agents found in configuration"}
|
||||
|
||||
agent = agents[0] if isinstance(agents, list) else agents
|
||||
except Exception as e:
|
||||
return {"error": f"Failed to load agents from {agent_file}: {e}"}
|
||||
else:
|
||||
return {"error": "Either --agent or --prompt must be provided"}
|
||||
|
||||
evaluator = AccuracyEvaluator(
|
||||
agent=agent,
|
||||
input_text=input_text,
|
||||
expected_output=expected_output,
|
||||
num_iterations=iterations,
|
||||
model=model,
|
||||
save_results_path=output_file,
|
||||
verbose=self.verbose
|
||||
)
|
||||
|
||||
result = evaluator.run(print_summary=True)
|
||||
return result.to_dict()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Accuracy evaluation failed: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
def run_performance(
|
||||
self,
|
||||
agent_file: str,
|
||||
input_text: str = "Hello",
|
||||
iterations: int = 10,
|
||||
warmup: int = 2,
|
||||
track_memory: bool = True,
|
||||
output_file: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Run performance evaluation on an agent.
|
||||
|
||||
Args:
|
||||
agent_file: Path to agents.yaml file
|
||||
input_text: Input to provide to the agent
|
||||
iterations: Number of benchmark iterations
|
||||
warmup: Number of warmup runs
|
||||
track_memory: Whether to track memory usage
|
||||
output_file: Path to save results
|
||||
|
||||
Returns:
|
||||
Evaluation result dictionary
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents.eval import PerformanceEvaluator
|
||||
from praisonai.agents_generator import AgentsGenerator
|
||||
except ImportError as e:
|
||||
logger.error(f"Failed to import evaluation modules: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
try:
|
||||
generator = AgentsGenerator(agent_file)
|
||||
agents = generator.generate_agents()
|
||||
|
||||
if not agents:
|
||||
return {"error": "No agents found in configuration"}
|
||||
|
||||
agent = agents[0] if isinstance(agents, list) else agents
|
||||
|
||||
evaluator = PerformanceEvaluator(
|
||||
agent=agent,
|
||||
input_text=input_text,
|
||||
num_iterations=iterations,
|
||||
warmup_runs=warmup,
|
||||
track_memory=track_memory,
|
||||
save_results_path=output_file,
|
||||
verbose=self.verbose
|
||||
)
|
||||
|
||||
result = evaluator.run(print_summary=True)
|
||||
return result.to_dict()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Performance evaluation failed: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
def run_reliability(
|
||||
self,
|
||||
agent_file: str,
|
||||
input_text: str,
|
||||
expected_tools: List[str],
|
||||
forbidden_tools: Optional[List[str]] = None,
|
||||
output_file: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Run reliability evaluation on an agent.
|
||||
|
||||
Args:
|
||||
agent_file: Path to agents.yaml file
|
||||
input_text: Input to provide to the agent
|
||||
expected_tools: List of tools that should be called
|
||||
forbidden_tools: List of tools that should NOT be called
|
||||
output_file: Path to save results
|
||||
|
||||
Returns:
|
||||
Evaluation result dictionary
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents.eval import ReliabilityEvaluator
|
||||
from praisonai.agents_generator import AgentsGenerator
|
||||
except ImportError as e:
|
||||
logger.error(f"Failed to import evaluation modules: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
try:
|
||||
generator = AgentsGenerator(agent_file)
|
||||
agents = generator.generate_agents()
|
||||
|
||||
if not agents:
|
||||
return {"error": "No agents found in configuration"}
|
||||
|
||||
agent = agents[0] if isinstance(agents, list) else agents
|
||||
|
||||
evaluator = ReliabilityEvaluator(
|
||||
agent=agent,
|
||||
input_text=input_text,
|
||||
expected_tools=expected_tools,
|
||||
forbidden_tools=forbidden_tools,
|
||||
save_results_path=output_file,
|
||||
verbose=self.verbose
|
||||
)
|
||||
|
||||
result = evaluator.run(print_summary=True)
|
||||
return result.to_dict()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Reliability evaluation failed: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
def run_criteria(
|
||||
self,
|
||||
agent_file: str,
|
||||
input_text: str,
|
||||
criteria: str,
|
||||
scoring_type: str = "numeric",
|
||||
threshold: float = 7.0,
|
||||
iterations: int = 1,
|
||||
model: Optional[str] = None,
|
||||
output_file: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Run criteria-based evaluation on an agent.
|
||||
|
||||
Args:
|
||||
agent_file: Path to agents.yaml file
|
||||
input_text: Input to provide to the agent
|
||||
criteria: Criteria to evaluate against
|
||||
scoring_type: "numeric" or "binary"
|
||||
threshold: Score threshold for passing (numeric mode)
|
||||
iterations: Number of evaluation iterations
|
||||
model: LLM model for judging
|
||||
output_file: Path to save results
|
||||
|
||||
Returns:
|
||||
Evaluation result dictionary
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents.eval import CriteriaEvaluator
|
||||
from praisonai.agents_generator import AgentsGenerator
|
||||
except ImportError as e:
|
||||
logger.error(f"Failed to import evaluation modules: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
try:
|
||||
generator = AgentsGenerator(agent_file)
|
||||
agents = generator.generate_agents()
|
||||
|
||||
if not agents:
|
||||
return {"error": "No agents found in configuration"}
|
||||
|
||||
agent = agents[0] if isinstance(agents, list) else agents
|
||||
|
||||
evaluator = CriteriaEvaluator(
|
||||
criteria=criteria,
|
||||
agent=agent,
|
||||
input_text=input_text,
|
||||
scoring_type=scoring_type,
|
||||
threshold=threshold,
|
||||
num_iterations=iterations,
|
||||
model=model,
|
||||
save_results_path=output_file,
|
||||
verbose=self.verbose
|
||||
)
|
||||
|
||||
result = evaluator.run(print_summary=True)
|
||||
return result.to_dict()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Criteria evaluation failed: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
def run_batch(
|
||||
self,
|
||||
agent_file: str,
|
||||
test_file: str,
|
||||
eval_type: str = "accuracy",
|
||||
output_file: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Run batch evaluation from a test file.
|
||||
|
||||
Args:
|
||||
agent_file: Path to agents.yaml file
|
||||
test_file: Path to JSON test file with test cases
|
||||
eval_type: Type of evaluation ("accuracy", "criteria")
|
||||
output_file: Path to save results
|
||||
|
||||
Returns:
|
||||
Batch evaluation results
|
||||
"""
|
||||
try:
|
||||
with open(test_file, 'r') as f:
|
||||
test_cases = json.load(f)
|
||||
except Exception as e:
|
||||
return {"error": f"Failed to load test file: {e}"}
|
||||
|
||||
results = []
|
||||
for i, test_case in enumerate(test_cases):
|
||||
if self.verbose:
|
||||
print(f"Running test case {i + 1}/{len(test_cases)}")
|
||||
|
||||
if eval_type == "accuracy":
|
||||
result = self.run_accuracy(
|
||||
agent_file=agent_file,
|
||||
input_text=test_case.get("input", ""),
|
||||
expected_output=test_case.get("expected", ""),
|
||||
iterations=test_case.get("iterations", 1)
|
||||
)
|
||||
elif eval_type == "criteria":
|
||||
result = self.run_criteria(
|
||||
agent_file=agent_file,
|
||||
input_text=test_case.get("input", ""),
|
||||
criteria=test_case.get("criteria", ""),
|
||||
scoring_type=test_case.get("scoring_type", "numeric"),
|
||||
threshold=test_case.get("threshold", 7.0)
|
||||
)
|
||||
else:
|
||||
result = {"error": f"Unknown eval type: {eval_type}"}
|
||||
|
||||
results.append({
|
||||
"test_case": i + 1,
|
||||
"input": test_case.get("input", ""),
|
||||
"result": result
|
||||
})
|
||||
|
||||
batch_result = {
|
||||
"total_tests": len(test_cases),
|
||||
"eval_type": eval_type,
|
||||
"results": results
|
||||
}
|
||||
|
||||
if output_file:
|
||||
try:
|
||||
with open(output_file, 'w') as f:
|
||||
json.dump(batch_result, f, indent=2)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save batch results: {e}")
|
||||
|
||||
return batch_result
|
||||
|
||||
|
||||
def handle_eval_command(args) -> int:
|
||||
"""
|
||||
Handle the eval CLI command.
|
||||
|
||||
Args:
|
||||
args: Command line arguments (list or parsed namespace)
|
||||
|
||||
Returns:
|
||||
Exit code
|
||||
"""
|
||||
import argparse
|
||||
|
||||
# If args is a list, parse it first
|
||||
if isinstance(args, list):
|
||||
parser = argparse.ArgumentParser(prog="praisonai eval")
|
||||
subparsers = parser.add_subparsers(dest='eval_type')
|
||||
add_eval_parser_subcommands(subparsers)
|
||||
|
||||
try:
|
||||
args = parser.parse_args(args)
|
||||
except SystemExit:
|
||||
return 1
|
||||
|
||||
if not args.eval_type:
|
||||
parser.print_help()
|
||||
print("\n[bold]Examples:[/bold]")
|
||||
print(" praisonai eval accuracy --prompt \"What is 2+2?\" --expected \"4\"")
|
||||
print(" praisonai eval performance --agent agents.yaml --input \"Hello\"")
|
||||
return 0
|
||||
|
||||
handler = EvalHandler(verbose=getattr(args, 'verbose', False))
|
||||
|
||||
eval_type = getattr(args, 'eval_type', 'accuracy')
|
||||
agent_file = getattr(args, 'agent', None)
|
||||
output_file = getattr(args, 'output', None)
|
||||
prompt = getattr(args, 'prompt', None)
|
||||
llm = getattr(args, 'llm', None)
|
||||
|
||||
# If no agent file and no prompt, check if agents.yaml exists
|
||||
if not agent_file and not prompt:
|
||||
import os
|
||||
if os.path.exists('agents.yaml'):
|
||||
agent_file = 'agents.yaml'
|
||||
|
||||
if eval_type == 'accuracy':
|
||||
result = handler.run_accuracy(
|
||||
agent_file=agent_file,
|
||||
input_text=getattr(args, 'input', ''),
|
||||
expected_output=getattr(args, 'expected', ''),
|
||||
iterations=getattr(args, 'iterations', 1),
|
||||
model=getattr(args, 'model', None),
|
||||
output_file=output_file,
|
||||
prompt=prompt,
|
||||
llm=llm
|
||||
)
|
||||
elif eval_type == 'performance':
|
||||
result = handler.run_performance(
|
||||
agent_file=agent_file,
|
||||
input_text=getattr(args, 'input', 'Hello'),
|
||||
iterations=getattr(args, 'iterations', 10),
|
||||
warmup=getattr(args, 'warmup', 2),
|
||||
track_memory=getattr(args, 'memory', True),
|
||||
output_file=output_file
|
||||
)
|
||||
elif eval_type == 'reliability':
|
||||
expected_tools = getattr(args, 'expected_tools', '').split(',')
|
||||
forbidden_tools = getattr(args, 'forbidden_tools', '')
|
||||
forbidden_tools = forbidden_tools.split(',') if forbidden_tools else None
|
||||
|
||||
result = handler.run_reliability(
|
||||
agent_file=agent_file,
|
||||
input_text=getattr(args, 'input', ''),
|
||||
expected_tools=expected_tools,
|
||||
forbidden_tools=forbidden_tools,
|
||||
output_file=output_file
|
||||
)
|
||||
elif eval_type == 'criteria':
|
||||
result = handler.run_criteria(
|
||||
agent_file=agent_file,
|
||||
input_text=getattr(args, 'input', ''),
|
||||
criteria=getattr(args, 'criteria', ''),
|
||||
scoring_type=getattr(args, 'scoring', 'numeric'),
|
||||
threshold=getattr(args, 'threshold', 7.0),
|
||||
iterations=getattr(args, 'iterations', 1),
|
||||
model=getattr(args, 'model', None),
|
||||
output_file=output_file
|
||||
)
|
||||
elif eval_type == 'batch':
|
||||
result = handler.run_batch(
|
||||
agent_file=agent_file,
|
||||
test_file=getattr(args, 'test_file', ''),
|
||||
eval_type=getattr(args, 'batch_type', 'accuracy'),
|
||||
output_file=output_file
|
||||
)
|
||||
else:
|
||||
print(f"Unknown evaluation type: {eval_type}")
|
||||
return 1
|
||||
|
||||
if 'error' in result:
|
||||
print(f"Error: {result['error']}")
|
||||
return 1
|
||||
elif not getattr(args, 'quiet', False):
|
||||
print(json.dumps(result, indent=2))
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def add_eval_parser_subcommands(subparsers) -> None:
|
||||
"""Add eval subcommand parsers to an existing subparsers object."""
|
||||
accuracy_parser = subparsers.add_parser('accuracy', help='Run accuracy evaluation')
|
||||
accuracy_parser.add_argument('--agent', '-a', help='Agent config file (optional if --prompt used)')
|
||||
accuracy_parser.add_argument('--prompt', '-p', type=str, help='Direct prompt (alternative to --agent)')
|
||||
accuracy_parser.add_argument('--llm', help='LLM model for agent (when using --prompt)')
|
||||
accuracy_parser.add_argument('--input', '-i', help='Input text (defaults to --prompt if not provided)')
|
||||
accuracy_parser.add_argument('--expected', '-e', required=True, help='Expected output')
|
||||
accuracy_parser.add_argument('--iterations', '-n', type=int, default=1, help='Number of iterations')
|
||||
accuracy_parser.add_argument('--model', '-m', help='Judge model')
|
||||
accuracy_parser.add_argument('--output', '-o', help='Output file')
|
||||
accuracy_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
|
||||
accuracy_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
|
||||
|
||||
perf_parser = subparsers.add_parser('performance', help='Run performance evaluation')
|
||||
perf_parser.add_argument('--agent', '-a', default='agents.yaml', help='Agent config file')
|
||||
perf_parser.add_argument('--input', '-i', default='Hello', help='Input text')
|
||||
perf_parser.add_argument('--iterations', '-n', type=int, default=10, help='Number of iterations')
|
||||
perf_parser.add_argument('--warmup', '-w', type=int, default=2, help='Warmup runs')
|
||||
perf_parser.add_argument('--memory', action='store_true', default=True, help='Track memory')
|
||||
perf_parser.add_argument('--output', '-o', help='Output file')
|
||||
perf_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
|
||||
perf_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
|
||||
|
||||
|
||||
def add_eval_parser(subparsers) -> None:
|
||||
"""
|
||||
Add eval subcommand parser.
|
||||
|
||||
Args:
|
||||
subparsers: Argument parser subparsers
|
||||
"""
|
||||
eval_parser = subparsers.add_parser(
|
||||
'eval',
|
||||
help='Run agent evaluations'
|
||||
)
|
||||
|
||||
eval_subparsers = eval_parser.add_subparsers(dest='eval_type')
|
||||
|
||||
accuracy_parser = eval_subparsers.add_parser(
|
||||
'accuracy',
|
||||
help='Run accuracy evaluation'
|
||||
)
|
||||
accuracy_parser.add_argument('--agent', '-a', help='Agent config file (optional if --prompt used)')
|
||||
accuracy_parser.add_argument('--prompt', '-p', help='Direct prompt (alternative to --agent)')
|
||||
accuracy_parser.add_argument('--llm', help='LLM model for agent (when using --prompt)')
|
||||
accuracy_parser.add_argument('--input', '-i', help='Input text (defaults to --prompt if not provided)')
|
||||
accuracy_parser.add_argument('--expected', '-e', required=True, help='Expected output')
|
||||
accuracy_parser.add_argument('--iterations', '-n', type=int, default=1, help='Number of iterations')
|
||||
accuracy_parser.add_argument('--model', '-m', help='Judge model')
|
||||
accuracy_parser.add_argument('--output', '-o', help='Output file')
|
||||
accuracy_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
|
||||
accuracy_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
|
||||
|
||||
perf_parser = eval_subparsers.add_parser(
|
||||
'performance',
|
||||
help='Run performance evaluation'
|
||||
)
|
||||
perf_parser.add_argument('--agent', '-a', default='agents.yaml', help='Agent config file')
|
||||
perf_parser.add_argument('--input', '-i', default='Hello', help='Input text')
|
||||
perf_parser.add_argument('--iterations', '-n', type=int, default=10, help='Number of iterations')
|
||||
perf_parser.add_argument('--warmup', '-w', type=int, default=2, help='Warmup runs')
|
||||
perf_parser.add_argument('--memory', action='store_true', default=True, help='Track memory')
|
||||
perf_parser.add_argument('--output', '-o', help='Output file')
|
||||
perf_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
|
||||
perf_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
|
||||
|
||||
rel_parser = eval_subparsers.add_parser(
|
||||
'reliability',
|
||||
help='Run reliability evaluation'
|
||||
)
|
||||
rel_parser.add_argument('--agent', '-a', default='agents.yaml', help='Agent config file')
|
||||
rel_parser.add_argument('--input', '-i', required=True, help='Input text')
|
||||
rel_parser.add_argument('--expected-tools', '-t', required=True, help='Expected tools (comma-separated)')
|
||||
rel_parser.add_argument('--forbidden-tools', '-f', help='Forbidden tools (comma-separated)')
|
||||
rel_parser.add_argument('--output', '-o', help='Output file')
|
||||
rel_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
|
||||
rel_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
|
||||
|
||||
criteria_parser = eval_subparsers.add_parser(
|
||||
'criteria',
|
||||
help='Run criteria-based evaluation'
|
||||
)
|
||||
criteria_parser.add_argument('--agent', '-a', default='agents.yaml', help='Agent config file')
|
||||
criteria_parser.add_argument('--input', '-i', required=True, help='Input text')
|
||||
criteria_parser.add_argument('--criteria', '-c', required=True, help='Evaluation criteria')
|
||||
criteria_parser.add_argument('--scoring', '-s', choices=['numeric', 'binary'], default='numeric', help='Scoring type')
|
||||
criteria_parser.add_argument('--threshold', type=float, default=7.0, help='Pass threshold')
|
||||
criteria_parser.add_argument('--iterations', '-n', type=int, default=1, help='Number of iterations')
|
||||
criteria_parser.add_argument('--model', '-m', help='Judge model')
|
||||
criteria_parser.add_argument('--output', '-o', help='Output file')
|
||||
criteria_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
|
||||
criteria_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
|
||||
|
||||
batch_parser = eval_subparsers.add_parser(
|
||||
'batch',
|
||||
help='Run batch evaluation from test file'
|
||||
)
|
||||
batch_parser.add_argument('--agent', '-a', default='agents.yaml', help='Agent config file')
|
||||
batch_parser.add_argument('--test-file', '-t', required=True, help='JSON test file')
|
||||
batch_parser.add_argument('--batch-type', '-b', choices=['accuracy', 'criteria'], default='accuracy', help='Evaluation type')
|
||||
batch_parser.add_argument('--output', '-o', help='Output file')
|
||||
batch_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
|
||||
batch_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
|
||||
@@ -3,12 +3,21 @@ Fast Context Handler for CLI.
|
||||
|
||||
Provides codebase search capability using FastContext.
|
||||
Usage: praisonai "Find authentication code" --fast-context ./src
|
||||
|
||||
Fast Context Strategy:
|
||||
1. Extract search keywords from natural language query using LLM
|
||||
2. Provide folder tree context to help understand codebase structure
|
||||
3. Use both file name search AND content search (ripgrep-style)
|
||||
4. Return relevant code snippets for context
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from typing import Any, Dict, Tuple, List
|
||||
from .base import FlagHandler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FastContextHandler(FlagHandler):
|
||||
"""
|
||||
@@ -61,14 +70,208 @@ class FastContextHandler(FlagHandler):
|
||||
|
||||
return True, ""
|
||||
|
||||
def _count_files_quick(self, path: str, max_count: int = 1000) -> int:
|
||||
"""Quickly count files in directory, stopping at max_count."""
|
||||
count = 0
|
||||
ignore_dirs = {'.git', '__pycache__', 'node_modules', '.venv', 'venv', '.cache', 'dist', 'build'}
|
||||
|
||||
for root, dirs, files in os.walk(path):
|
||||
dirs[:] = [d for d in dirs if d not in ignore_dirs and not d.startswith('.')]
|
||||
count += len(files)
|
||||
if count >= max_count:
|
||||
return count
|
||||
return count
|
||||
|
||||
def _get_folder_tree(self, path: str, max_depth: int = 2) -> str:
|
||||
"""
|
||||
Get a compact folder tree structure for context.
|
||||
|
||||
Optimized for large directories:
|
||||
- Limits depth to max_depth (default 2)
|
||||
- Shows max 10 files per directory
|
||||
- Truncates total output to 50 lines
|
||||
- Skips common non-code directories
|
||||
"""
|
||||
# Quick check for very large directories
|
||||
file_count = self._count_files_quick(path, 500)
|
||||
if file_count >= 500:
|
||||
# For large directories, reduce depth and show summary
|
||||
max_depth = 1
|
||||
logger.debug(f"Large directory detected ({file_count}+ files), reducing tree depth")
|
||||
|
||||
tree_lines = []
|
||||
ignore_dirs = {'.git', '__pycache__', 'node_modules', '.venv', 'venv', '.cache', 'dist', 'build', 'coverage', '.pytest_cache', '.mypy_cache', 'eggs', '*.egg-info'}
|
||||
ignore_exts = {'.pyc', '.pyo', '.so', '.o', '.a', '.dylib', '.class', '.jar'}
|
||||
total_files = 0
|
||||
max_total_files = 100 # Stop after seeing this many files
|
||||
|
||||
def walk_tree(current_path, prefix="", depth=0):
|
||||
nonlocal total_files
|
||||
if depth > max_depth or total_files >= max_total_files:
|
||||
return
|
||||
|
||||
try:
|
||||
entries = sorted(os.listdir(current_path))
|
||||
except PermissionError:
|
||||
return
|
||||
|
||||
dirs = []
|
||||
files = []
|
||||
for entry in entries:
|
||||
full_path = os.path.join(current_path, entry)
|
||||
if os.path.isdir(full_path):
|
||||
if entry not in ignore_dirs and not entry.startswith('.'):
|
||||
dirs.append(entry)
|
||||
else:
|
||||
ext = os.path.splitext(entry)[1]
|
||||
if ext not in ignore_exts and not entry.startswith('.'):
|
||||
files.append(entry)
|
||||
|
||||
# Limit directories shown at each level
|
||||
dirs_to_show = dirs[:15]
|
||||
for i, d in enumerate(dirs_to_show):
|
||||
is_last = (i == len(dirs_to_show) - 1) and not files
|
||||
tree_lines.append(f"{prefix}{'└── ' if is_last else '├── '}{d}/")
|
||||
walk_tree(os.path.join(current_path, d),
|
||||
prefix + (' ' if is_last else '│ '), depth + 1)
|
||||
if len(dirs) > 15:
|
||||
tree_lines.append(f"{prefix} ... and {len(dirs) - 15} more directories")
|
||||
|
||||
# Limit files shown
|
||||
files_to_show = files[:10]
|
||||
total_files += len(files)
|
||||
for i, f in enumerate(files_to_show):
|
||||
is_last = i == len(files_to_show) - 1
|
||||
tree_lines.append(f"{prefix}{'└── ' if is_last else '├── '}{f}")
|
||||
if len(files) > 10:
|
||||
tree_lines.append(f"{prefix} ... and {len(files) - 10} more files")
|
||||
|
||||
tree_lines.append(os.path.basename(path) + "/")
|
||||
walk_tree(path)
|
||||
return "\n".join(tree_lines[:50]) # Limit total lines
|
||||
|
||||
def _extract_keywords(self, query: str, folder_tree: str) -> List[str]:
|
||||
"""Extract search keywords from natural language query using LLM."""
|
||||
try:
|
||||
from praisonaiagents import Agent
|
||||
|
||||
extractor = Agent(
|
||||
name="KeywordExtractor",
|
||||
role="Search Query Analyzer",
|
||||
goal="Extract file search keywords from natural language",
|
||||
backstory="Expert at understanding code search queries",
|
||||
llm="gpt-4o-mini",
|
||||
verbose=False
|
||||
)
|
||||
|
||||
prompt = f"""Given this folder structure:
|
||||
{folder_tree}
|
||||
|
||||
And this search query: "{query}"
|
||||
|
||||
Extract 1-5 specific keywords/patterns to search for in file names and content.
|
||||
Return ONLY the keywords, one per line, no explanations.
|
||||
Focus on: file names, function names, class names, variable names that might match.
|
||||
If the query mentions a folder name that exists, include files from that folder."""
|
||||
|
||||
response = extractor.chat(prompt, stream=False)
|
||||
keywords = [k.strip() for k in str(response).strip().split('\n') if k.strip()]
|
||||
logger.debug(f"Extracted keywords: {keywords}")
|
||||
return keywords[:5] # Limit to 5 keywords
|
||||
except Exception as e:
|
||||
logger.debug(f"Keyword extraction failed: {e}")
|
||||
# Fallback: extract words from query
|
||||
return [w for w in query.split() if len(w) > 2][:3]
|
||||
|
||||
def _search_files_by_name(self, path: str, keywords: List[str]) -> List[Dict[str, Any]]:
|
||||
"""Search for files matching keywords in their names."""
|
||||
matches = []
|
||||
path = os.path.abspath(path)
|
||||
|
||||
ignore_dirs = {'.git', '__pycache__', 'node_modules', '.venv', 'venv'}
|
||||
|
||||
for root, dirs, files in os.walk(path):
|
||||
dirs[:] = [d for d in dirs if d not in ignore_dirs]
|
||||
|
||||
for filename in files:
|
||||
filepath = os.path.join(root, filename)
|
||||
rel_path = os.path.relpath(filepath, path)
|
||||
|
||||
# Check if any keyword matches filename or path
|
||||
for kw in keywords:
|
||||
kw_lower = kw.lower()
|
||||
if kw_lower in filename.lower() or kw_lower in rel_path.lower():
|
||||
matches.append({
|
||||
'file': rel_path,
|
||||
'lines': [(1, 50)], # First 50 lines
|
||||
'relevance': 1.0,
|
||||
'match_type': 'filename'
|
||||
})
|
||||
break
|
||||
|
||||
return matches[:20] # Limit results
|
||||
|
||||
def _search_files_by_content(self, path: str, keywords: List[str]) -> List[Dict[str, Any]]:
|
||||
"""Search for files containing keywords in content."""
|
||||
matches = []
|
||||
path = os.path.abspath(path)
|
||||
|
||||
ignore_dirs = {'.git', '__pycache__', 'node_modules', '.venv', 'venv'}
|
||||
code_exts = {'.py', '.js', '.ts', '.jsx', '.tsx', '.java', '.go', '.rs', '.rb', '.sh', '.yaml', '.yml', '.json', '.md', '.txt'}
|
||||
|
||||
for root, dirs, files in os.walk(path):
|
||||
dirs[:] = [d for d in dirs if d not in ignore_dirs]
|
||||
|
||||
for filename in files:
|
||||
ext = os.path.splitext(filename)[1]
|
||||
if ext not in code_exts:
|
||||
continue
|
||||
|
||||
filepath = os.path.join(root, filename)
|
||||
rel_path = os.path.relpath(filepath, path)
|
||||
|
||||
try:
|
||||
with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
|
||||
content = f.read(50000) # Read first 50KB
|
||||
|
||||
content_lower = content.lower()
|
||||
for kw in keywords:
|
||||
if kw.lower() in content_lower:
|
||||
# Find line numbers with matches
|
||||
lines = content.split('\n')
|
||||
match_lines = []
|
||||
for i, line in enumerate(lines[:200], 1): # Check first 200 lines
|
||||
if kw.lower() in line.lower():
|
||||
match_lines.append((max(1, i-2), min(len(lines), i+2)))
|
||||
|
||||
if match_lines:
|
||||
matches.append({
|
||||
'file': rel_path,
|
||||
'lines': match_lines[:5], # Limit line ranges
|
||||
'relevance': 0.8,
|
||||
'match_type': 'content'
|
||||
})
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return matches[:15] # Limit results
|
||||
|
||||
def search_context(self, query: str, path: str, **kwargs) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search for relevant context in the codebase.
|
||||
|
||||
Uses a smart multi-step approach:
|
||||
1. Get folder tree structure
|
||||
2. Use LLM to extract search keywords from query
|
||||
3. Search by filename AND content
|
||||
4. Return combined results
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
query: Natural language search query
|
||||
path: Path to search
|
||||
**kwargs: Additional search options
|
||||
- use_llm_keywords: Use LLM to extract keywords (default: True)
|
||||
|
||||
Returns:
|
||||
List of matching results
|
||||
@@ -84,32 +287,50 @@ class FastContextHandler(FlagHandler):
|
||||
return []
|
||||
|
||||
try:
|
||||
from praisonaiagents import FastContext
|
||||
use_llm_keywords = kwargs.get('use_llm_keywords', True)
|
||||
|
||||
fc = FastContext(
|
||||
path=path,
|
||||
model=kwargs.get('model', 'gpt-4o-mini'),
|
||||
max_turns=kwargs.get('max_turns', 4),
|
||||
parallelism=kwargs.get('parallelism', 8),
|
||||
timeout=kwargs.get('timeout', 30.0)
|
||||
)
|
||||
# Step 1: Get folder tree for context
|
||||
logger.debug(f"Getting folder tree for: {path}")
|
||||
folder_tree = self._get_folder_tree(path)
|
||||
logger.debug(f"Folder tree:\n{folder_tree}")
|
||||
|
||||
result = fc.search(query)
|
||||
# Step 2: Extract keywords
|
||||
if use_llm_keywords:
|
||||
logger.debug(f"Extracting keywords from query: {query}")
|
||||
keywords = self._extract_keywords(query, folder_tree)
|
||||
else:
|
||||
keywords = [w for w in query.split() if len(w) > 2][:3]
|
||||
|
||||
# Convert to list of dicts
|
||||
logger.debug(f"Search keywords: {keywords}")
|
||||
|
||||
if not keywords:
|
||||
self.print_status("Could not extract search keywords", "warning")
|
||||
return []
|
||||
|
||||
# Step 3: Search by filename
|
||||
logger.debug("Searching by filename...")
|
||||
filename_matches = self._search_files_by_name(path, keywords)
|
||||
logger.debug(f"Found {len(filename_matches)} filename matches")
|
||||
|
||||
# Step 4: Search by content
|
||||
logger.debug("Searching by content...")
|
||||
content_matches = self._search_files_by_content(path, keywords)
|
||||
logger.debug(f"Found {len(content_matches)} content matches")
|
||||
|
||||
# Combine and deduplicate
|
||||
seen_files = set()
|
||||
matches = []
|
||||
if hasattr(result, 'files'):
|
||||
for file_match in result.files:
|
||||
matches.append({
|
||||
'file': file_match.path,
|
||||
'lines': [(lr.start, lr.end) for lr in file_match.line_ranges],
|
||||
'relevance': getattr(file_match, 'relevance', 1.0)
|
||||
})
|
||||
|
||||
for m in filename_matches + content_matches:
|
||||
if m['file'] not in seen_files:
|
||||
seen_files.add(m['file'])
|
||||
matches.append(m)
|
||||
|
||||
self.print_status(f"🔍 Found {len(matches)} relevant files", "success")
|
||||
return matches
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"FastContext search error: {e}")
|
||||
self.log(f"FastContext search error: {e}", "error")
|
||||
return []
|
||||
|
||||
|
||||
@@ -75,7 +75,7 @@ class GuardrailHandler(FlagHandler):
|
||||
|
||||
Args:
|
||||
description: Natural language description of validation criteria
|
||||
llm: Optional LLM instance for validation
|
||||
llm: Optional LLM instance for validation (defaults to gpt-4o-mini)
|
||||
|
||||
Returns:
|
||||
LLMGuardrail instance or None if unavailable
|
||||
@@ -87,6 +87,12 @@ class GuardrailHandler(FlagHandler):
|
||||
)
|
||||
return None
|
||||
|
||||
# Provide default LLM if none specified
|
||||
if llm is None:
|
||||
import os
|
||||
# Try to get from environment or use default
|
||||
llm = os.environ.get('OPENAI_MODEL_NAME', 'gpt-4o-mini')
|
||||
|
||||
return LLMGuardrail(description=description, llm=llm)
|
||||
|
||||
def validate_output(self, output: str, description: str, llm: Any = None) -> Tuple[bool, str]:
|
||||
|
||||
@@ -72,11 +72,11 @@ class HandoffHandler(FlagHandler):
|
||||
self.print_status(msg, "error")
|
||||
return []
|
||||
|
||||
from praisonaiagents import Agent, Handoff
|
||||
from praisonaiagents import Agent, handoff
|
||||
|
||||
agents = []
|
||||
|
||||
# Create agents for each role
|
||||
# First pass: Create all agents without handoffs
|
||||
for i, name in enumerate(agent_names):
|
||||
agent_config = {
|
||||
"name": name.title().replace("_", " "),
|
||||
@@ -91,26 +91,45 @@ class HandoffHandler(FlagHandler):
|
||||
agent = Agent(**agent_config)
|
||||
agents.append(agent)
|
||||
|
||||
# Set up handoffs between agents
|
||||
for i, agent in enumerate(agents[:-1]):
|
||||
next_agent = agents[i + 1]
|
||||
handoff = Handoff(
|
||||
target=next_agent,
|
||||
name=f"handoff_to_{agent_names[i + 1]}",
|
||||
description=f"Hand off task to {agent_names[i + 1]} for further processing"
|
||||
)
|
||||
# Second pass: Set up handoffs by recreating agents with handoff config
|
||||
final_agents = []
|
||||
for i, name in enumerate(agent_names):
|
||||
agent_config = {
|
||||
"name": name.title().replace("_", " "),
|
||||
"role": name.title().replace("_", " "),
|
||||
"goal": f"Complete tasks as {name}",
|
||||
"backstory": f"Expert {name} with specialized skills"
|
||||
}
|
||||
|
||||
# Add handoff to agent
|
||||
if not hasattr(agent, 'handoffs') or agent.handoffs is None:
|
||||
agent.handoffs = []
|
||||
agent.handoffs.append(handoff)
|
||||
if llm:
|
||||
agent_config["llm"] = llm
|
||||
|
||||
# Add handoff to next agent if not the last agent
|
||||
if i < len(agent_names) - 1:
|
||||
next_agent = agents[i + 1]
|
||||
next_agent_name = agent_names[i + 1]
|
||||
|
||||
# Pass handoffs via constructor - this properly registers them as tools
|
||||
agent_config["handoffs"] = [
|
||||
handoff(
|
||||
agent=next_agent,
|
||||
tool_name_override=f"handoff_to_{next_agent_name}",
|
||||
tool_description_override=f"Hand off the task to {next_agent_name}. Call this after completing your part."
|
||||
)
|
||||
]
|
||||
|
||||
# Add instructions to use handoff
|
||||
agent_config["instructions"] = f"After completing your task, you MUST use the handoff_to_{next_agent_name} tool to pass your work to the next agent."
|
||||
|
||||
final_agent = Agent(**agent_config)
|
||||
final_agents.append(final_agent)
|
||||
|
||||
self.print_status(f"🤝 Created {len(agents)} agents with handoff chain", "success")
|
||||
self.print_status(f"🤝 Created {len(final_agents)} agents with handoff chain", "success")
|
||||
for i, name in enumerate(agent_names):
|
||||
arrow = " → " if i < len(agent_names) - 1 else ""
|
||||
self.print_status(f" {name}{arrow}", "info")
|
||||
|
||||
return agents
|
||||
return final_agents
|
||||
|
||||
def apply_to_agent_config(self, config: Dict[str, Any], flag_value: Any) -> Dict[str, Any]:
|
||||
"""
|
||||
|
||||
@@ -1,27 +1,47 @@
|
||||
"""
|
||||
Image Handler for CLI.
|
||||
|
||||
Provides image processing capabilities using ImageAgent.
|
||||
Usage: praisonai "Describe this image" --image path/to/image.png
|
||||
Provides two distinct image capabilities:
|
||||
1. Image Description (Vision) - Analyze existing images using vision-capable LLMs
|
||||
Usage: praisonai "Describe this image" --image path/to/image.png
|
||||
|
||||
2. Image Generation - Generate new images from text prompts
|
||||
Usage: praisonai "A sunset over mountains" --image-generate
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, Tuple, List
|
||||
import base64
|
||||
import logging
|
||||
from typing import Any, Dict, Tuple, List, Optional
|
||||
from .base import FlagHandler
|
||||
|
||||
# Supported image extensions
|
||||
SUPPORTED_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.svg'}
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Supported image extensions for vision/description
|
||||
SUPPORTED_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.svg', '.tiff'}
|
||||
|
||||
# Default vision-capable models (gpt-4o has better vision support than gpt-4o-mini)
|
||||
DEFAULT_VISION_MODEL = "gpt-4o"
|
||||
|
||||
# Default image generation models
|
||||
DEFAULT_IMAGE_GEN_MODEL = "dall-e-3"
|
||||
SUPPORTED_IMAGE_GEN_MODELS = [
|
||||
'gpt-image-1', 'gpt-image-1-mini', 'gpt-image-1.5',
|
||||
'chatgpt-image-latest', 'dall-e-2', 'dall-e-3'
|
||||
]
|
||||
|
||||
|
||||
class ImageHandler(FlagHandler):
|
||||
"""
|
||||
Handler for --image flag.
|
||||
Handler for --image flag (Image Description/Vision).
|
||||
|
||||
Processes images using ImageAgent for vision-based tasks.
|
||||
Describes/analyzes existing images using vision-capable LLMs like GPT-4o.
|
||||
This is for understanding what's IN an image, not generating new images.
|
||||
|
||||
Example:
|
||||
praisonai "Describe this image" --image photo.png
|
||||
praisonai "What's in this image?" --image /path/to/image.jpg
|
||||
praisonai "Count the objects" --image image1.png,image2.png
|
||||
"""
|
||||
|
||||
@property
|
||||
@@ -34,17 +54,15 @@ class ImageHandler(FlagHandler):
|
||||
|
||||
@property
|
||||
def flag_help(self) -> str:
|
||||
return "Path to image file for vision-based tasks"
|
||||
return "Path to image file for vision-based description/analysis"
|
||||
|
||||
def check_dependencies(self) -> Tuple[bool, str]:
|
||||
"""Check if ImageAgent is available."""
|
||||
"""Check if litellm is available for vision."""
|
||||
try:
|
||||
import importlib.util
|
||||
if importlib.util.find_spec("praisonaiagents") is not None:
|
||||
return True, ""
|
||||
return False, "praisonaiagents not installed. Install with: pip install praisonaiagents"
|
||||
import litellm
|
||||
return True, ""
|
||||
except ImportError:
|
||||
return False, "praisonaiagents not installed. Install with: pip install praisonaiagents"
|
||||
return False, "litellm not installed. Install with: pip install litellm"
|
||||
|
||||
def validate_image_path(self, path: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
@@ -71,127 +89,154 @@ class ImageHandler(FlagHandler):
|
||||
|
||||
return True, ""
|
||||
|
||||
def _encode_image_to_base64(self, image_path: str) -> str:
|
||||
"""Encode a local image file to base64."""
|
||||
with open(image_path, 'rb') as f:
|
||||
return base64.b64encode(f.read()).decode('utf-8')
|
||||
|
||||
def _get_image_mime_type(self, image_path: str) -> str:
|
||||
"""Get MIME type based on file extension."""
|
||||
ext = os.path.splitext(image_path)[1].lower()
|
||||
mime_types = {
|
||||
'.png': 'image/png',
|
||||
'.jpg': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.gif': 'image/gif',
|
||||
'.webp': 'image/webp',
|
||||
'.bmp': 'image/bmp',
|
||||
'.svg': 'image/svg+xml',
|
||||
'.tiff': 'image/tiff',
|
||||
}
|
||||
return mime_types.get(ext, 'image/png')
|
||||
|
||||
def _build_image_content(self, image_path: str) -> Dict[str, Any]:
|
||||
"""Build image content for vision API."""
|
||||
if image_path.startswith(('http://', 'https://')):
|
||||
return {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_path}
|
||||
}
|
||||
else:
|
||||
mime_type = self._get_image_mime_type(image_path)
|
||||
base64_data = self._encode_image_to_base64(image_path)
|
||||
return {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:{mime_type};base64,{base64_data}"}
|
||||
}
|
||||
|
||||
def apply_to_agent_config(self, config: Dict[str, Any], flag_value: Any) -> Dict[str, Any]:
|
||||
"""
|
||||
Apply image configuration to agent config.
|
||||
|
||||
Args:
|
||||
config: Agent configuration dictionary
|
||||
flag_value: Image path or list of paths
|
||||
|
||||
Returns:
|
||||
Modified configuration with image settings
|
||||
"""
|
||||
"""Apply image configuration to agent config."""
|
||||
if flag_value:
|
||||
# Store image path(s) for use with ImageAgent
|
||||
if isinstance(flag_value, str):
|
||||
config["images"] = [flag_value]
|
||||
elif isinstance(flag_value, list):
|
||||
config["images"] = flag_value
|
||||
config["use_image_agent"] = True
|
||||
config["use_vision"] = True
|
||||
return config
|
||||
|
||||
def create_image_agent(self, llm: str = None, **kwargs) -> Any:
|
||||
def describe_image(self, prompt: str, image_path: str, llm = None) -> str:
|
||||
"""
|
||||
Create an ImageAgent instance.
|
||||
|
||||
Args:
|
||||
llm: Optional LLM model name
|
||||
**kwargs: Additional agent configuration
|
||||
|
||||
Returns:
|
||||
ImageAgent instance or None if unavailable
|
||||
"""
|
||||
available, msg = self.check_dependencies()
|
||||
if not available:
|
||||
self.print_status(msg, "error")
|
||||
return None
|
||||
|
||||
from praisonaiagents import ImageAgent
|
||||
|
||||
agent_config = {
|
||||
"name": "ImageAnalyzer",
|
||||
"role": "Image Analyst",
|
||||
"goal": "Analyze and describe images accurately",
|
||||
"backstory": "Expert at understanding visual content"
|
||||
}
|
||||
|
||||
if llm:
|
||||
agent_config["llm"] = llm
|
||||
|
||||
agent_config.update(kwargs)
|
||||
|
||||
return ImageAgent(**agent_config)
|
||||
|
||||
def process_image(self, prompt: str, image_path: str, llm: str = None) -> str:
|
||||
"""
|
||||
Process an image with the given prompt.
|
||||
Describe an image using vision-capable LLM.
|
||||
|
||||
Args:
|
||||
prompt: The prompt/question about the image
|
||||
image_path: Path to the image file
|
||||
llm: Optional LLM model name
|
||||
llm: Optional LLM model name (must be vision-capable), can be string or dict
|
||||
|
||||
Returns:
|
||||
Agent's response about the image
|
||||
Description of the image
|
||||
"""
|
||||
# Validate image path
|
||||
valid, msg = self.validate_image_path(image_path)
|
||||
if not valid:
|
||||
return f"Error: {msg}"
|
||||
|
||||
agent = self.create_image_agent(llm=llm)
|
||||
if not agent:
|
||||
return "Error: Could not create ImageAgent"
|
||||
available, dep_msg = self.check_dependencies()
|
||||
if not available:
|
||||
return f"Error: {dep_msg}"
|
||||
|
||||
import litellm
|
||||
|
||||
# Handle llm being a dict (from agent_config) or string
|
||||
if isinstance(llm, dict):
|
||||
model = llm.get('model', DEFAULT_VISION_MODEL)
|
||||
else:
|
||||
model = llm or DEFAULT_VISION_MODEL
|
||||
|
||||
try:
|
||||
# ImageAgent expects images as a list
|
||||
result = agent.chat(prompt, images=[image_path])
|
||||
return result
|
||||
# Build message with image
|
||||
content = [
|
||||
{"type": "text", "text": prompt},
|
||||
self._build_image_content(image_path)
|
||||
]
|
||||
|
||||
response = litellm.completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": content}]
|
||||
)
|
||||
|
||||
return response.choices[0].message.content
|
||||
|
||||
except Exception as e:
|
||||
self.log(f"Image processing error: {e}", "error")
|
||||
return f"Error processing image: {e}"
|
||||
logger.error(f"Error describing image: {e}")
|
||||
return f"Error describing image: {e}"
|
||||
|
||||
def process_multiple_images(self, prompt: str, image_paths: List[str], llm: str = None) -> str:
|
||||
def describe_multiple_images(self, prompt: str, image_paths: List[str], llm = None) -> str:
|
||||
"""
|
||||
Process multiple images with the given prompt.
|
||||
Describe multiple images using vision-capable LLM.
|
||||
|
||||
Args:
|
||||
prompt: The prompt/question about the images
|
||||
image_paths: List of paths to image files
|
||||
llm: Optional LLM model name
|
||||
llm: Optional LLM model name, can be string or dict
|
||||
|
||||
Returns:
|
||||
Agent's response about the images
|
||||
Description of the images
|
||||
"""
|
||||
# Validate all image paths
|
||||
for path in image_paths:
|
||||
valid, msg = self.validate_image_path(path)
|
||||
if not valid:
|
||||
return f"Error: {msg}"
|
||||
|
||||
agent = self.create_image_agent(llm=llm)
|
||||
if not agent:
|
||||
return "Error: Could not create ImageAgent"
|
||||
available, dep_msg = self.check_dependencies()
|
||||
if not available:
|
||||
return f"Error: {dep_msg}"
|
||||
|
||||
import litellm
|
||||
|
||||
# Handle llm being a dict (from agent_config) or string
|
||||
if isinstance(llm, dict):
|
||||
model = llm.get('model', DEFAULT_VISION_MODEL)
|
||||
else:
|
||||
model = llm or DEFAULT_VISION_MODEL
|
||||
|
||||
try:
|
||||
result = agent.chat(prompt, images=image_paths)
|
||||
return result
|
||||
# Build message with all images
|
||||
content = [{"type": "text", "text": prompt}]
|
||||
for path in image_paths:
|
||||
content.append(self._build_image_content(path))
|
||||
|
||||
response = litellm.completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": content}]
|
||||
)
|
||||
|
||||
return response.choices[0].message.content
|
||||
|
||||
except Exception as e:
|
||||
self.log(f"Image processing error: {e}", "error")
|
||||
return f"Error processing images: {e}"
|
||||
logger.error(f"Error describing images: {e}")
|
||||
return f"Error describing images: {e}"
|
||||
|
||||
def execute(self, prompt: str = None, image_path: str = None, llm: str = None, **kwargs) -> str:
|
||||
"""
|
||||
Execute image processing.
|
||||
Execute image description (vision).
|
||||
|
||||
Args:
|
||||
prompt: The prompt/question about the image
|
||||
image_path: Path to the image file (or comma-separated paths)
|
||||
llm: Optional LLM model name
|
||||
llm: Optional vision-capable LLM model name
|
||||
|
||||
Returns:
|
||||
Agent's response
|
||||
Description of the image(s)
|
||||
"""
|
||||
if not prompt:
|
||||
return "Error: No prompt provided"
|
||||
@@ -202,6 +247,138 @@ class ImageHandler(FlagHandler):
|
||||
# Handle multiple images (comma-separated)
|
||||
if ',' in image_path:
|
||||
paths = [p.strip() for p in image_path.split(',')]
|
||||
return self.process_multiple_images(prompt, paths, llm)
|
||||
return self.describe_multiple_images(prompt, paths, llm)
|
||||
|
||||
return self.process_image(prompt, image_path, llm)
|
||||
return self.describe_image(prompt, image_path, llm)
|
||||
|
||||
|
||||
class ImageGenerateHandler(FlagHandler):
|
||||
"""
|
||||
Handler for --image-generate flag (Image Generation).
|
||||
|
||||
Generates NEW images from text prompts using DALL-E or similar models.
|
||||
This is for CREATING images, not analyzing existing ones.
|
||||
|
||||
Example:
|
||||
praisonai "A sunset over mountains" --image-generate
|
||||
praisonai "A cat wearing a hat" --image-generate --llm dall-e-3
|
||||
"""
|
||||
|
||||
@property
|
||||
def feature_name(self) -> str:
|
||||
return "image_generate"
|
||||
|
||||
@property
|
||||
def flag_name(self) -> str:
|
||||
return "image_generate"
|
||||
|
||||
@property
|
||||
def flag_help(self) -> str:
|
||||
return "Generate an image from the text prompt"
|
||||
|
||||
def check_dependencies(self) -> Tuple[bool, str]:
|
||||
"""Check if ImageAgent is available."""
|
||||
try:
|
||||
import importlib.util
|
||||
if importlib.util.find_spec("praisonaiagents") is not None:
|
||||
return True, ""
|
||||
return False, "praisonaiagents not installed. Install with: pip install praisonaiagents"
|
||||
except ImportError:
|
||||
return False, "praisonaiagents not installed. Install with: pip install praisonaiagents"
|
||||
|
||||
def apply_to_agent_config(self, config: Dict[str, Any], flag_value: Any) -> Dict[str, Any]:
|
||||
"""Apply image generation configuration."""
|
||||
if flag_value:
|
||||
config["use_image_generation"] = True
|
||||
return config
|
||||
|
||||
def generate_image(self, prompt: str, llm: str = None, output_path: str = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Generate an image from a text prompt.
|
||||
|
||||
Args:
|
||||
prompt: Text description of the image to generate
|
||||
llm: Image generation model (dall-e-3, dall-e-2, etc.)
|
||||
output_path: Optional path to save the generated image
|
||||
|
||||
Returns:
|
||||
Dict with image URL or path, or error message
|
||||
"""
|
||||
available, msg = self.check_dependencies()
|
||||
if not available:
|
||||
return {"error": msg}
|
||||
|
||||
from praisonaiagents import ImageAgent
|
||||
|
||||
# Use provided model or default
|
||||
model = llm or DEFAULT_IMAGE_GEN_MODEL
|
||||
|
||||
# Validate model is supported for generation
|
||||
if model not in SUPPORTED_IMAGE_GEN_MODELS:
|
||||
return {
|
||||
"error": f"Invalid model: '{model}'. Supported image generation models: {', '.join(SUPPORTED_IMAGE_GEN_MODELS)}"
|
||||
}
|
||||
|
||||
try:
|
||||
agent = ImageAgent(
|
||||
name="ImageGenerator",
|
||||
role="Image Generation Assistant",
|
||||
goal="Generate high-quality images from text descriptions",
|
||||
llm=model
|
||||
)
|
||||
|
||||
result = agent.chat(prompt)
|
||||
|
||||
# Save image if output path provided
|
||||
if output_path and isinstance(result, dict) and 'data' in result:
|
||||
self._save_image(result, output_path)
|
||||
result['saved_to'] = output_path
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating image: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
def _save_image(self, result: Dict[str, Any], output_path: str) -> bool:
|
||||
"""Save generated image to file."""
|
||||
try:
|
||||
import requests
|
||||
|
||||
if 'data' in result and len(result['data']) > 0:
|
||||
image_data = result['data'][0]
|
||||
|
||||
if 'url' in image_data:
|
||||
# Download from URL
|
||||
response = requests.get(image_data['url'])
|
||||
with open(output_path, 'wb') as f:
|
||||
f.write(response.content)
|
||||
return True
|
||||
elif 'b64_json' in image_data:
|
||||
# Decode base64
|
||||
import base64
|
||||
img_bytes = base64.b64decode(image_data['b64_json'])
|
||||
with open(output_path, 'wb') as f:
|
||||
f.write(img_bytes)
|
||||
return True
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Error saving image: {e}")
|
||||
return False
|
||||
|
||||
def execute(self, prompt: str = None, llm: str = None, output_path: str = None, **kwargs) -> Dict[str, Any]:
|
||||
"""
|
||||
Execute image generation.
|
||||
|
||||
Args:
|
||||
prompt: Text description of the image to generate
|
||||
llm: Image generation model name
|
||||
output_path: Optional path to save the image
|
||||
|
||||
Returns:
|
||||
Dict with generated image info or error
|
||||
"""
|
||||
if not prompt:
|
||||
return {"error": "No prompt provided for image generation"}
|
||||
|
||||
return self.generate_image(prompt, llm, output_path)
|
||||
@@ -107,6 +107,14 @@ class MetricsHandler(FlagHandler):
|
||||
if hasattr(agent, 'last_token_metrics'):
|
||||
metrics.update(agent.last_token_metrics or {})
|
||||
|
||||
# Check llm_model property for model info
|
||||
if hasattr(agent, 'llm_model'):
|
||||
llm_model = agent.llm_model
|
||||
if hasattr(llm_model, 'model'):
|
||||
metrics['model'] = llm_model.model
|
||||
elif isinstance(llm_model, str):
|
||||
metrics['model'] = llm_model
|
||||
|
||||
if hasattr(agent, 'llm') and agent.llm:
|
||||
llm = agent.llm
|
||||
if hasattr(llm, 'last_token_metrics'):
|
||||
@@ -117,6 +125,42 @@ class MetricsHandler(FlagHandler):
|
||||
if hasattr(agent, 'metrics') and isinstance(agent.metrics, dict):
|
||||
metrics.update(agent.metrics or {})
|
||||
|
||||
# Try to get from litellm's success callback data
|
||||
try:
|
||||
import litellm
|
||||
# Check if there's accumulated usage data
|
||||
if hasattr(litellm, '_thread_context') and litellm._thread_context:
|
||||
ctx = litellm._thread_context
|
||||
if hasattr(ctx, 'usage'):
|
||||
usage = ctx.usage
|
||||
if hasattr(usage, 'prompt_tokens'):
|
||||
metrics['prompt_tokens'] = usage.prompt_tokens
|
||||
if hasattr(usage, 'completion_tokens'):
|
||||
metrics['completion_tokens'] = usage.completion_tokens
|
||||
if hasattr(usage, 'total_tokens'):
|
||||
metrics['total_tokens'] = usage.total_tokens
|
||||
except:
|
||||
pass
|
||||
|
||||
# Try to get cost from litellm cost tracking
|
||||
try:
|
||||
import litellm
|
||||
from litellm import completion_cost
|
||||
# If we have token counts, estimate cost
|
||||
if 'prompt_tokens' in metrics and 'completion_tokens' in metrics:
|
||||
model = metrics.get('model', 'gpt-4o-mini')
|
||||
try:
|
||||
cost = completion_cost(
|
||||
model=model,
|
||||
prompt_tokens=metrics['prompt_tokens'],
|
||||
completion_tokens=metrics['completion_tokens']
|
||||
)
|
||||
metrics['cost'] = cost
|
||||
except:
|
||||
pass
|
||||
except:
|
||||
pass
|
||||
|
||||
return metrics
|
||||
|
||||
def post_process_result(self, result: Any, flag_value: Any) -> Any:
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
"""
|
||||
Profile Handler for CLI.
|
||||
|
||||
Provides performance profiling and benchmarking capabilities.
|
||||
Usage: praisonai profile <subcommand> [options]
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import time
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProfileResult:
|
||||
"""Result from a profiling run."""
|
||||
prompt: str
|
||||
response: str
|
||||
execution_time_ms: float
|
||||
model: str
|
||||
tokens: Optional[Dict[str, int]] = None
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"prompt": self.prompt,
|
||||
"response": self.response[:200] + "..." if len(self.response) > 200 else self.response,
|
||||
"execution_time_ms": self.execution_time_ms,
|
||||
"model": self.model,
|
||||
"tokens": self.tokens,
|
||||
"timestamp": self.timestamp
|
||||
}
|
||||
|
||||
|
||||
def run_benchmark(prompt: str, model: str = None, iterations: int = 1, verbose: bool = False) -> List[ProfileResult]:
|
||||
"""
|
||||
Run a benchmark on a prompt.
|
||||
|
||||
Args:
|
||||
prompt: The prompt to benchmark
|
||||
model: Optional model to use
|
||||
iterations: Number of iterations to run
|
||||
verbose: Enable verbose output
|
||||
|
||||
Returns:
|
||||
List of ProfileResult objects
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents import Agent
|
||||
except ImportError:
|
||||
print("[red]ERROR: praisonaiagents not installed. Install with: pip install praisonaiagents[/red]")
|
||||
return []
|
||||
|
||||
results = []
|
||||
|
||||
agent_config = {
|
||||
"name": "BenchmarkAgent",
|
||||
"role": "Benchmark Assistant",
|
||||
"goal": "Complete tasks efficiently",
|
||||
"verbose": verbose
|
||||
}
|
||||
|
||||
if model:
|
||||
agent_config["llm"] = model
|
||||
|
||||
agent = Agent(**agent_config)
|
||||
|
||||
for i in range(iterations):
|
||||
if verbose:
|
||||
print(f"[dim]Running iteration {i + 1}/{iterations}...[/dim]")
|
||||
|
||||
start_time = time.time()
|
||||
response = agent.start(prompt)
|
||||
end_time = time.time()
|
||||
|
||||
execution_time_ms = (end_time - start_time) * 1000
|
||||
|
||||
result = ProfileResult(
|
||||
prompt=prompt,
|
||||
response=str(response),
|
||||
execution_time_ms=execution_time_ms,
|
||||
model=model or "default"
|
||||
)
|
||||
results.append(result)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def display_benchmark_results(results: List[ProfileResult], output_format: str = "table"):
|
||||
"""Display benchmark results."""
|
||||
if not results:
|
||||
print("[yellow]No results to display[/yellow]")
|
||||
return
|
||||
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
console = Console()
|
||||
except ImportError:
|
||||
# Fallback to simple output
|
||||
for r in results:
|
||||
print(f"Time: {r.execution_time_ms:.1f}ms | Model: {r.model}")
|
||||
return
|
||||
|
||||
if output_format == "json":
|
||||
print(json.dumps([r.to_dict() for r in results], indent=2))
|
||||
return
|
||||
|
||||
# Calculate statistics
|
||||
times = [r.execution_time_ms for r in results]
|
||||
avg_time = sum(times) / len(times)
|
||||
min_time = min(times)
|
||||
max_time = max(times)
|
||||
|
||||
table = Table(title="📊 Benchmark Results")
|
||||
table.add_column("Metric", style="cyan")
|
||||
table.add_column("Value", style="green")
|
||||
|
||||
table.add_row("Iterations", str(len(results)))
|
||||
table.add_row("Average Time", f"{avg_time:.1f}ms")
|
||||
table.add_row("Min Time", f"{min_time:.1f}ms")
|
||||
table.add_row("Max Time", f"{max_time:.1f}ms")
|
||||
table.add_row("Model", results[0].model)
|
||||
|
||||
console.print(table)
|
||||
|
||||
# Show response preview
|
||||
if results:
|
||||
console.print("\n[bold]Response Preview:[/bold]")
|
||||
preview = results[0].response[:300] + "..." if len(results[0].response) > 300 else results[0].response
|
||||
console.print(f"[dim]{preview}[/dim]")
|
||||
|
||||
|
||||
def handle_profile_command(args: List[str]) -> int:
|
||||
"""
|
||||
Handle the profile command.
|
||||
|
||||
Args:
|
||||
args: Command line arguments
|
||||
|
||||
Returns:
|
||||
Exit code
|
||||
"""
|
||||
parser = argparse.ArgumentParser(prog="praisonai profile", description="Performance profiling and benchmarking")
|
||||
subparsers = parser.add_subparsers(dest='subcommand', help='Profile subcommands')
|
||||
|
||||
# Benchmark subcommand
|
||||
bench_parser = subparsers.add_parser('benchmark', aliases=['bench', 'run'], help='Run a benchmark')
|
||||
bench_parser.add_argument('prompt', nargs='?', help='Prompt to benchmark')
|
||||
bench_parser.add_argument('--model', '-m', type=str, help='Model to use')
|
||||
bench_parser.add_argument('--iterations', '-n', type=int, default=1, help='Number of iterations')
|
||||
bench_parser.add_argument('--output', '-o', type=str, help='Output file path')
|
||||
bench_parser.add_argument('--format', '-f', type=str, choices=['table', 'json', 'html'], default='table', help='Output format')
|
||||
bench_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
|
||||
|
||||
# Compare subcommand
|
||||
compare_parser = subparsers.add_parser('compare', help='Compare models')
|
||||
compare_parser.add_argument('prompt', nargs='?', help='Prompt to compare')
|
||||
compare_parser.add_argument('--models', type=str, help='Comma-separated list of models to compare')
|
||||
|
||||
# Report subcommand
|
||||
report_parser = subparsers.add_parser('report', help='Generate a profiling report')
|
||||
report_parser.add_argument('--input', '-i', type=str, help='Input JSON file with results')
|
||||
report_parser.add_argument('--output', '-o', type=str, help='Output file path')
|
||||
report_parser.add_argument('--format', '-f', type=str, choices=['html', 'json', 'markdown'], default='html', help='Report format')
|
||||
|
||||
try:
|
||||
parsed_args = parser.parse_args(args)
|
||||
except SystemExit:
|
||||
return 1
|
||||
|
||||
if not parsed_args.subcommand:
|
||||
parser.print_help()
|
||||
print("\n[bold]Examples:[/bold]")
|
||||
print(" praisonai profile benchmark \"What is 2+2?\"")
|
||||
print(" praisonai profile benchmark \"Write a poem\" --model gpt-4o --iterations 3")
|
||||
print(" praisonai profile compare \"Explain AI\" --models gpt-4o,gpt-4o-mini")
|
||||
return 0
|
||||
|
||||
if parsed_args.subcommand in ['benchmark', 'bench', 'run']:
|
||||
if not parsed_args.prompt:
|
||||
print("[red]ERROR: Please provide a prompt to benchmark[/red]")
|
||||
print("Usage: praisonai profile benchmark \"Your prompt here\"")
|
||||
return 1
|
||||
|
||||
print(f"[bold cyan]🔬 Running benchmark...[/bold cyan]")
|
||||
results = run_benchmark(
|
||||
parsed_args.prompt,
|
||||
model=parsed_args.model,
|
||||
iterations=parsed_args.iterations,
|
||||
verbose=parsed_args.verbose
|
||||
)
|
||||
|
||||
display_benchmark_results(results, parsed_args.format)
|
||||
|
||||
if parsed_args.output:
|
||||
with open(parsed_args.output, 'w') as f:
|
||||
json.dump([r.to_dict() for r in results], f, indent=2)
|
||||
print(f"[green]Results saved to {parsed_args.output}[/green]")
|
||||
|
||||
return 0
|
||||
|
||||
elif parsed_args.subcommand == 'compare':
|
||||
if not parsed_args.prompt:
|
||||
print("[red]ERROR: Please provide a prompt to compare[/red]")
|
||||
return 1
|
||||
|
||||
models = parsed_args.models.split(',') if parsed_args.models else ['gpt-4o-mini', 'gpt-4o']
|
||||
print(f"[bold cyan]🔬 Comparing models: {', '.join(models)}[/bold cyan]")
|
||||
|
||||
all_results = []
|
||||
for model in models:
|
||||
print(f"[dim]Testing {model}...[/dim]")
|
||||
results = run_benchmark(parsed_args.prompt, model=model, iterations=1)
|
||||
all_results.extend(results)
|
||||
|
||||
display_benchmark_results(all_results)
|
||||
return 0
|
||||
|
||||
elif parsed_args.subcommand == 'report':
|
||||
print("[yellow]Report generation not yet implemented[/yellow]")
|
||||
return 0
|
||||
|
||||
return 0
|
||||
@@ -0,0 +1,350 @@
|
||||
"""
|
||||
PraisonAI CLI Profiling Commands
|
||||
|
||||
Provides CLI commands for profiling agent execution.
|
||||
|
||||
Usage:
|
||||
praisonai profile run "task" --output report.html
|
||||
praisonai profile report --format json
|
||||
praisonai profile benchmark --iterations 10
|
||||
"""
|
||||
|
||||
import time
|
||||
import json
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
|
||||
class ProfilingHandler:
|
||||
"""
|
||||
Handler for CLI profiling commands.
|
||||
|
||||
Features:
|
||||
- Run tasks with profiling enabled
|
||||
- Generate reports (console, JSON, HTML)
|
||||
- Benchmark agent performance
|
||||
- Export flamegraphs
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._profiler = None
|
||||
|
||||
@property
|
||||
def profiler(self):
|
||||
"""Lazy load profiler."""
|
||||
if self._profiler is None:
|
||||
from praisonai.profiler import Profiler
|
||||
self._profiler = Profiler
|
||||
return self._profiler
|
||||
|
||||
def enable(self) -> None:
|
||||
"""Enable profiling."""
|
||||
self.profiler.enable()
|
||||
|
||||
def disable(self) -> None:
|
||||
"""Disable profiling."""
|
||||
self.profiler.disable()
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear profiling data."""
|
||||
self.profiler.clear()
|
||||
|
||||
def run_with_profiling(self, task: str, config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Run a task with profiling enabled.
|
||||
|
||||
Args:
|
||||
task: Task description or prompt
|
||||
config: Optional configuration dict
|
||||
|
||||
Returns:
|
||||
Dict with result and profiling data
|
||||
"""
|
||||
self.profiler.enable()
|
||||
self.profiler.clear()
|
||||
|
||||
result = None
|
||||
error = None
|
||||
|
||||
try:
|
||||
# Import and run PraisonAI
|
||||
from praisonai import PraisonAI
|
||||
|
||||
with self.profiler.block("total_execution"):
|
||||
with self.profiler.block("initialization"):
|
||||
praison = PraisonAI(auto=task, **(config or {}))
|
||||
|
||||
with self.profiler.block("agent_run"):
|
||||
result = praison.run()
|
||||
except Exception as e:
|
||||
error = str(e)
|
||||
finally:
|
||||
self.profiler.disable()
|
||||
|
||||
return {
|
||||
'result': result,
|
||||
'error': error,
|
||||
'summary': self.profiler.get_summary(),
|
||||
'statistics': self.profiler.get_statistics()
|
||||
}
|
||||
|
||||
def report(self, format: str = "console", output: Optional[str] = None) -> str:
|
||||
"""
|
||||
Generate profiling report.
|
||||
|
||||
Args:
|
||||
format: Output format (console, json, html)
|
||||
output: Optional file path to save report
|
||||
|
||||
Returns:
|
||||
Report content as string
|
||||
"""
|
||||
if format == "console":
|
||||
content = self.profiler.report(output="console")
|
||||
elif format == "json":
|
||||
content = self.profiler.export_json()
|
||||
elif format == "html":
|
||||
content = self.profiler.export_html()
|
||||
else:
|
||||
raise ValueError(f"Unknown format: {format}")
|
||||
|
||||
if output:
|
||||
with open(output, 'w') as f:
|
||||
f.write(content)
|
||||
print(f"Report saved to: {output}")
|
||||
|
||||
return content
|
||||
|
||||
def benchmark(self, task: str, iterations: int = 5,
|
||||
warmup: int = 1, config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Benchmark agent performance.
|
||||
|
||||
Args:
|
||||
task: Task to benchmark
|
||||
iterations: Number of iterations
|
||||
warmup: Number of warmup runs
|
||||
config: Optional configuration
|
||||
|
||||
Returns:
|
||||
Benchmark results with statistics
|
||||
"""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
times = []
|
||||
errors = []
|
||||
|
||||
# Warmup runs
|
||||
for i in range(warmup):
|
||||
print(f"Warmup {i + 1}/{warmup}...")
|
||||
try:
|
||||
self.run_with_profiling(task, config)
|
||||
except Exception as e:
|
||||
print(f"Warmup error: {e}")
|
||||
|
||||
# Benchmark runs
|
||||
for i in range(iterations):
|
||||
print(f"Iteration {i + 1}/{iterations}...")
|
||||
Profiler.clear()
|
||||
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
self.run_with_profiling(task, config)
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
times.append(duration_ms)
|
||||
except Exception as e:
|
||||
errors.append(str(e))
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
times.append(duration_ms)
|
||||
|
||||
# Calculate statistics
|
||||
if times:
|
||||
sorted_times = sorted(times)
|
||||
n = len(sorted_times)
|
||||
|
||||
results = {
|
||||
'iterations': iterations,
|
||||
'successful': iterations - len(errors),
|
||||
'failed': len(errors),
|
||||
'times_ms': times,
|
||||
'mean_ms': sum(times) / n,
|
||||
'min_ms': min(times),
|
||||
'max_ms': max(times),
|
||||
'p50_ms': sorted_times[n // 2],
|
||||
'p95_ms': sorted_times[int(n * 0.95)] if n > 1 else sorted_times[-1],
|
||||
'p99_ms': sorted_times[int(n * 0.99)] if n > 1 else sorted_times[-1],
|
||||
'errors': errors
|
||||
}
|
||||
else:
|
||||
results = {
|
||||
'iterations': iterations,
|
||||
'successful': 0,
|
||||
'failed': len(errors),
|
||||
'errors': errors
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
def export_flamegraph(self, filepath: str) -> None:
|
||||
"""Export flamegraph to SVG file."""
|
||||
self.profiler.export_flamegraph(filepath)
|
||||
print(f"Flamegraph saved to: {filepath}")
|
||||
|
||||
def get_summary(self) -> Dict[str, Any]:
|
||||
"""Get profiling summary."""
|
||||
return self.profiler.get_summary()
|
||||
|
||||
def get_statistics(self, category: Optional[str] = None) -> Dict[str, float]:
|
||||
"""Get profiling statistics."""
|
||||
return self.profiler.get_statistics(category)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print profiling summary to console."""
|
||||
summary = self.get_summary()
|
||||
stats = self.get_statistics()
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("PROFILING SUMMARY")
|
||||
print("=" * 60)
|
||||
|
||||
print(f"\nTotal Time: {summary['total_time_ms']:.2f}ms")
|
||||
print(f"Operations: {summary['timing_count']}")
|
||||
print(f"Imports: {summary['import_count']}")
|
||||
print(f"Flow Steps: {summary['flow_steps']}")
|
||||
|
||||
print("\nStatistics:")
|
||||
print(f" P50 (Median): {stats['p50']:.2f}ms")
|
||||
print(f" P95: {stats['p95']:.2f}ms")
|
||||
print(f" P99: {stats['p99']:.2f}ms")
|
||||
print(f" Mean: {stats['mean']:.2f}ms")
|
||||
print(f" Std Dev: {stats['std_dev']:.2f}ms")
|
||||
|
||||
print("\nSlowest Operations:")
|
||||
for name, duration in summary['slowest_operations'][:5]:
|
||||
print(f" {name}: {duration:.2f}ms")
|
||||
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
def handle_profile_command(args) -> int:
|
||||
"""
|
||||
Handle profile CLI command.
|
||||
|
||||
Args:
|
||||
args: Parsed command line arguments
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success)
|
||||
"""
|
||||
handler = ProfilingHandler()
|
||||
|
||||
subcommand = getattr(args, 'profile_command', None)
|
||||
|
||||
if subcommand == 'run':
|
||||
task = getattr(args, 'task', '')
|
||||
output = getattr(args, 'output', None)
|
||||
format_type = getattr(args, 'format', 'console')
|
||||
|
||||
print(f"Running with profiling: {task[:50]}...")
|
||||
result = handler.run_with_profiling(task)
|
||||
|
||||
if result['error']:
|
||||
print(f"Error: {result['error']}")
|
||||
|
||||
handler.report(format=format_type, output=output)
|
||||
return 0 if not result['error'] else 1
|
||||
|
||||
elif subcommand == 'report':
|
||||
output = getattr(args, 'output', None)
|
||||
format_type = getattr(args, 'format', 'console')
|
||||
|
||||
handler.report(format=format_type, output=output)
|
||||
return 0
|
||||
|
||||
elif subcommand == 'benchmark':
|
||||
task = getattr(args, 'task', '')
|
||||
iterations = getattr(args, 'iterations', 5)
|
||||
warmup = getattr(args, 'warmup', 1)
|
||||
output = getattr(args, 'output', None)
|
||||
|
||||
print(f"Benchmarking: {task[:50]}...")
|
||||
results = handler.benchmark(task, iterations=iterations, warmup=warmup)
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("BENCHMARK RESULTS")
|
||||
print("=" * 60)
|
||||
print(f"Iterations: {results['iterations']}")
|
||||
print(f"Successful: {results['successful']}")
|
||||
print(f"Failed: {results['failed']}")
|
||||
|
||||
if 'mean_ms' in results:
|
||||
print("\nTiming:")
|
||||
print(f" Mean: {results['mean_ms']:.2f}ms")
|
||||
print(f" Min: {results['min_ms']:.2f}ms")
|
||||
print(f" Max: {results['max_ms']:.2f}ms")
|
||||
print(f" P50: {results['p50_ms']:.2f}ms")
|
||||
print(f" P95: {results['p95_ms']:.2f}ms")
|
||||
|
||||
if output:
|
||||
with open(output, 'w') as f:
|
||||
json.dump(results, f, indent=2)
|
||||
print(f"\nResults saved to: {output}")
|
||||
|
||||
return 0
|
||||
|
||||
elif subcommand == 'flamegraph':
|
||||
output = getattr(args, 'output', 'profile.svg')
|
||||
handler.export_flamegraph(output)
|
||||
return 0
|
||||
|
||||
elif subcommand == 'summary':
|
||||
handler.print_summary()
|
||||
return 0
|
||||
|
||||
else:
|
||||
print("Usage: praisonai profile <run|report|benchmark|flamegraph|summary>")
|
||||
return 1
|
||||
|
||||
|
||||
def add_profile_parser(subparsers) -> None:
|
||||
"""Add profile subcommand to argument parser."""
|
||||
profile_parser = subparsers.add_parser(
|
||||
'profile',
|
||||
help='Profile agent execution'
|
||||
)
|
||||
|
||||
profile_subparsers = profile_parser.add_subparsers(dest='profile_command')
|
||||
|
||||
# Run with profiling
|
||||
run_parser = profile_subparsers.add_parser('run', help='Run task with profiling')
|
||||
run_parser.add_argument('task', help='Task to run')
|
||||
run_parser.add_argument('--output', '-o', help='Output file path')
|
||||
run_parser.add_argument('--format', '-f', choices=['console', 'json', 'html'],
|
||||
default='console', help='Output format')
|
||||
|
||||
# Generate report
|
||||
report_parser = profile_subparsers.add_parser('report', help='Generate profiling report')
|
||||
report_parser.add_argument('--output', '-o', help='Output file path')
|
||||
report_parser.add_argument('--format', '-f', choices=['console', 'json', 'html'],
|
||||
default='console', help='Output format')
|
||||
|
||||
# Benchmark
|
||||
bench_parser = profile_subparsers.add_parser('benchmark', help='Benchmark agent performance')
|
||||
bench_parser.add_argument('task', help='Task to benchmark')
|
||||
bench_parser.add_argument('--iterations', '-n', type=int, default=5, help='Number of iterations')
|
||||
bench_parser.add_argument('--warmup', '-w', type=int, default=1, help='Warmup runs')
|
||||
bench_parser.add_argument('--output', '-o', help='Output file for results')
|
||||
|
||||
# Flamegraph
|
||||
flame_parser = profile_subparsers.add_parser('flamegraph', help='Export flamegraph')
|
||||
flame_parser.add_argument('--output', '-o', default='profile.svg', help='Output SVG file')
|
||||
|
||||
# Summary
|
||||
profile_subparsers.add_parser('summary', help='Print profiling summary') # noqa: F841
|
||||
|
||||
|
||||
# Exports
|
||||
__all__ = [
|
||||
'ProfilingHandler',
|
||||
'handle_profile_command',
|
||||
'add_profile_parser',
|
||||
]
|
||||
@@ -0,0 +1,839 @@
|
||||
"""
|
||||
Skills CLI Feature Handler
|
||||
|
||||
Provides CLI commands for managing Agent Skills:
|
||||
- list: List available skills
|
||||
- validate: Validate a skill directory
|
||||
- create: Create a new skill from template
|
||||
- prompt: Generate prompt XML for skills
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional, List
|
||||
|
||||
|
||||
class SkillsHandler:
|
||||
"""Handler for skills CLI commands."""
|
||||
|
||||
def __init__(self, verbose: bool = True):
|
||||
"""Initialize the skills handler.
|
||||
|
||||
Args:
|
||||
verbose: Whether to print verbose output
|
||||
"""
|
||||
self.verbose = verbose
|
||||
|
||||
def list_skills(
|
||||
self,
|
||||
skill_dirs: Optional[List[str]] = None,
|
||||
include_defaults: bool = True
|
||||
) -> List[dict]:
|
||||
"""List all available skills.
|
||||
|
||||
Args:
|
||||
skill_dirs: Optional list of directories to scan
|
||||
include_defaults: Whether to include default skill directories
|
||||
|
||||
Returns:
|
||||
List of skill info dictionaries
|
||||
"""
|
||||
from praisonaiagents.skills import discover_skills
|
||||
|
||||
skills = discover_skills(skill_dirs, include_defaults)
|
||||
|
||||
result = []
|
||||
for skill in skills:
|
||||
info = {
|
||||
"name": skill.name,
|
||||
"description": skill.description,
|
||||
"path": str(skill.path) if skill.path else None,
|
||||
"license": skill.license,
|
||||
}
|
||||
result.append(info)
|
||||
|
||||
if self.verbose:
|
||||
print(f" {skill.name}: {skill.description[:60]}...")
|
||||
|
||||
if self.verbose and not result:
|
||||
print("No skills found.")
|
||||
|
||||
return result
|
||||
|
||||
def validate_skill(self, skill_path: str) -> dict:
|
||||
"""Validate a skill directory.
|
||||
|
||||
Args:
|
||||
skill_path: Path to the skill directory
|
||||
|
||||
Returns:
|
||||
Validation result dictionary with 'valid' and 'errors' keys
|
||||
"""
|
||||
from praisonaiagents.skills import validate
|
||||
|
||||
path = Path(skill_path).expanduser().resolve()
|
||||
errors = validate(path)
|
||||
|
||||
result = {
|
||||
"valid": len(errors) == 0,
|
||||
"path": str(path),
|
||||
"errors": errors
|
||||
}
|
||||
|
||||
if self.verbose:
|
||||
if result["valid"]:
|
||||
print(f"✓ Skill at {path} is valid")
|
||||
else:
|
||||
print(f"✗ Skill at {path} has errors:")
|
||||
for error in errors:
|
||||
print(f" - {error}")
|
||||
|
||||
return result
|
||||
|
||||
def create_skill(
|
||||
self,
|
||||
name: str,
|
||||
description: str = "A custom skill",
|
||||
output_dir: Optional[str] = None,
|
||||
author: Optional[str] = None,
|
||||
license: Optional[str] = None,
|
||||
compatibility: Optional[str] = None,
|
||||
template: bool = False,
|
||||
use_ai: bool = True,
|
||||
generate_script: bool = False
|
||||
) -> str:
|
||||
"""Create a new skill from template or AI generation.
|
||||
|
||||
Args:
|
||||
name: Skill name (kebab-case)
|
||||
description: Skill description (also used as prompt for AI)
|
||||
output_dir: Directory to create skill in (default: current dir)
|
||||
author: Author name for metadata
|
||||
license: License type (default: Apache-2.0)
|
||||
compatibility: Compatibility information
|
||||
template: If True, use template only (no AI)
|
||||
use_ai: If True, try to use AI to generate content
|
||||
generate_script: If True, generate scripts/skill.py
|
||||
|
||||
Returns:
|
||||
Path to created skill directory
|
||||
"""
|
||||
# Validate name format
|
||||
import re
|
||||
if not re.match(r'^[a-z][a-z0-9-]*[a-z0-9]$', name) and len(name) > 1:
|
||||
if not re.match(r'^[a-z]$', name):
|
||||
raise ValueError(
|
||||
f"Invalid skill name '{name}'. "
|
||||
"Must be lowercase, use hyphens, and not start/end with hyphen."
|
||||
)
|
||||
|
||||
base_dir = Path(output_dir or os.getcwd())
|
||||
skill_dir = base_dir / name
|
||||
|
||||
if skill_dir.exists():
|
||||
raise ValueError(f"Directory already exists: {skill_dir}")
|
||||
|
||||
# Create directory structure
|
||||
skill_dir.mkdir(parents=True)
|
||||
(skill_dir / "scripts").mkdir()
|
||||
(skill_dir / "references").mkdir()
|
||||
(skill_dir / "assets").mkdir()
|
||||
|
||||
# Set defaults
|
||||
author = author or "user"
|
||||
license = license or "Apache-2.0"
|
||||
compatibility = compatibility or "Works with PraisonAI Agents"
|
||||
|
||||
# Try AI generation if requested and not in template mode
|
||||
ai_content = None
|
||||
if use_ai and not template:
|
||||
ai_content = self._generate_skill_content_with_ai(name, description)
|
||||
|
||||
if ai_content and ai_content.get("skill_md"):
|
||||
# Use AI-generated content
|
||||
skill_md_content = ai_content["skill_md"]
|
||||
if self.verbose:
|
||||
print("✓ Generated SKILL.md with AI")
|
||||
else:
|
||||
# Use template
|
||||
skill_md_content = self._generate_template_content(
|
||||
name, description, author, license, compatibility,
|
||||
include_script=generate_script
|
||||
)
|
||||
if self.verbose and use_ai and not template:
|
||||
print("⚠ No API key found, using template")
|
||||
|
||||
(skill_dir / "SKILL.md").write_text(skill_md_content)
|
||||
|
||||
# Generate scripts/script.py if requested or AI provided it
|
||||
if generate_script or (ai_content and ai_content.get("skill_py")):
|
||||
script_content = (ai_content or {}).get("skill_py") or self._generate_template_script(name, description)
|
||||
# Strip any remaining code blocks from script content
|
||||
script_content = self._strip_code_blocks(script_content)
|
||||
(skill_dir / "scripts" / "script.py").write_text(script_content)
|
||||
if self.verbose:
|
||||
print(" - scripts/script.py")
|
||||
else:
|
||||
(skill_dir / "scripts" / ".gitkeep").write_text("")
|
||||
|
||||
# Create placeholder files
|
||||
(skill_dir / "references" / ".gitkeep").write_text("")
|
||||
(skill_dir / "assets" / ".gitkeep").write_text("")
|
||||
|
||||
if self.verbose:
|
||||
print(f"✓ Created skill at {skill_dir}")
|
||||
print(" - SKILL.md")
|
||||
print(" - scripts/")
|
||||
print(" - references/")
|
||||
print(" - assets/")
|
||||
|
||||
return str(skill_dir)
|
||||
|
||||
def _generate_template_content(
|
||||
self,
|
||||
name: str,
|
||||
description: str,
|
||||
author: str,
|
||||
license: str,
|
||||
compatibility: str,
|
||||
include_script: bool = False
|
||||
) -> str:
|
||||
"""Generate template SKILL.md content."""
|
||||
title = name.replace('-', ' ').title()
|
||||
func_name = name.replace('-', '_')
|
||||
|
||||
script_section = ""
|
||||
if include_script:
|
||||
script_section = f"""
|
||||
## Script Usage
|
||||
|
||||
This skill includes a Python script at `scripts/script.py` that provides the core functionality.
|
||||
|
||||
### Running the Script
|
||||
|
||||
```bash
|
||||
python scripts/script.py <input>
|
||||
```
|
||||
|
||||
### Using as a Module
|
||||
|
||||
```python
|
||||
from scripts.script import {func_name}
|
||||
|
||||
result = {func_name}(input_data)
|
||||
print(result)
|
||||
```
|
||||
"""
|
||||
|
||||
return f"""---
|
||||
name: {name}
|
||||
description: {description}
|
||||
license: {license}
|
||||
compatibility: {compatibility}
|
||||
metadata:
|
||||
author: {author}
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
# {title}
|
||||
|
||||
## Overview
|
||||
|
||||
{description}
|
||||
{script_section}
|
||||
## Usage
|
||||
|
||||
Describe how to use this skill.
|
||||
|
||||
## Instructions
|
||||
|
||||
1. Step one
|
||||
2. Step two
|
||||
3. Step three
|
||||
"""
|
||||
|
||||
def _generate_template_script(self, name: str, description: str) -> str:
|
||||
"""Generate description-relevant scripts/script.py content."""
|
||||
func_name = name.replace('-', '_')
|
||||
|
||||
# Analyze description to generate relevant code
|
||||
desc_lower = description.lower()
|
||||
|
||||
# CSV/Data analysis patterns
|
||||
if any(kw in desc_lower for kw in ['csv', 'spreadsheet', 'data analysis', 'analyze data', 'tabular']):
|
||||
return self._generate_csv_script(name, description, func_name)
|
||||
|
||||
# PDF patterns
|
||||
if any(kw in desc_lower for kw in ['pdf', 'document', 'extract text']):
|
||||
return self._generate_pdf_script(name, description, func_name)
|
||||
|
||||
# Web/API patterns
|
||||
if any(kw in desc_lower for kw in ['api', 'http', 'request', 'web', 'fetch', 'url']):
|
||||
return self._generate_api_script(name, description, func_name)
|
||||
|
||||
# File processing patterns
|
||||
if any(kw in desc_lower for kw in ['file', 'read', 'write', 'process', 'parse']):
|
||||
return self._generate_file_script(name, description, func_name)
|
||||
|
||||
# Image patterns
|
||||
if any(kw in desc_lower for kw in ['image', 'photo', 'picture', 'resize', 'convert']):
|
||||
return self._generate_image_script(name, description, func_name)
|
||||
|
||||
# JSON/YAML patterns
|
||||
if any(kw in desc_lower for kw in ['json', 'yaml', 'config', 'configuration']):
|
||||
return self._generate_json_script(name, description, func_name)
|
||||
|
||||
# Text processing patterns
|
||||
if any(kw in desc_lower for kw in ['text', 'string', 'regex', 'search', 'replace', 'format']):
|
||||
return self._generate_text_script(name, description, func_name)
|
||||
|
||||
# Default generic script
|
||||
return self._generate_generic_script(name, description, func_name)
|
||||
|
||||
def _generate_csv_script(self, name: str, description: str, func_name: str) -> str:
|
||||
"""Generate CSV analysis script."""
|
||||
return f'''"""
|
||||
{name} - {description}
|
||||
|
||||
This script provides functionality for the {name} skill.
|
||||
"""
|
||||
|
||||
def {func_name}(file_path: str) -> dict:
|
||||
"""
|
||||
Analyze CSV file and return statistics.
|
||||
|
||||
Args:
|
||||
file_path: Path to the CSV file to analyze
|
||||
|
||||
Returns:
|
||||
Dictionary with analysis results
|
||||
"""
|
||||
import pandas as pd
|
||||
|
||||
df = pd.read_csv(file_path)
|
||||
|
||||
result = {{
|
||||
"rows": len(df),
|
||||
"columns": len(df.columns),
|
||||
"column_names": list(df.columns),
|
||||
"dtypes": {{col: str(dtype) for col, dtype in df.dtypes.items()}},
|
||||
"missing_values": df.isnull().sum().to_dict(),
|
||||
"numeric_summary": {{}}
|
||||
}}
|
||||
|
||||
numeric_cols = df.select_dtypes(include=['number']).columns
|
||||
for col in numeric_cols:
|
||||
result["numeric_summary"][col] = {{
|
||||
"mean": float(df[col].mean()),
|
||||
"std": float(df[col].std()),
|
||||
"min": float(df[col].min()),
|
||||
"max": float(df[col].max())
|
||||
}}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point for the skill."""
|
||||
import sys
|
||||
import json
|
||||
|
||||
if len(sys.argv) > 1:
|
||||
result = {func_name}(sys.argv[1])
|
||||
print(json.dumps(result, indent=2))
|
||||
else:
|
||||
print("Usage: python script.py <csv_file>")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
'''
|
||||
|
||||
def _generate_pdf_script(self, name: str, description: str, func_name: str) -> str:
|
||||
"""Generate PDF processing script."""
|
||||
return f'''"""\n{name} - {description}\n\nThis script provides PDF processing functionality.\n"""\n\ndef {func_name}(file_path: str) -> dict:\n """\n Process PDF file and extract content.\n \n Args:\n file_path: Path to the PDF file\n \n Returns:\n Dictionary with extracted content\n """\n from pypdf import PdfReader\n \n reader = PdfReader(file_path)\n \n result = {{\n "pages": len(reader.pages),\n "metadata": {{\n "title": reader.metadata.title if reader.metadata else None,\n "author": reader.metadata.author if reader.metadata else None\n }},\n "text": []\n }}\n \n for i, page in enumerate(reader.pages):\n result["text"].append({{\n "page": i + 1,\n "content": page.extract_text()\n }})\n \n return result\n\n\ndef main():\n """Main entry point for the skill."""\n import sys\n import json\n \n if len(sys.argv) > 1:\n result = {func_name}(sys.argv[1])\n print(json.dumps(result, indent=2))\n else:\n print("Usage: python script.py <pdf_file>")\n\n\nif __name__ == "__main__":\n main()\n'''
|
||||
|
||||
def _generate_api_script(self, name: str, description: str, func_name: str) -> str:
|
||||
"""Generate API/HTTP request script."""
|
||||
return f'''"""\n{name} - {description}\n\nThis script provides API request functionality.\n"""\nimport requests\n\ndef {func_name}(url: str, method: str = "GET", data: dict = None) -> dict:\n """\n Make HTTP request to API endpoint.\n \n Args:\n url: API endpoint URL\n method: HTTP method (GET, POST, PUT, DELETE)\n data: Optional request body data\n \n Returns:\n Dictionary with response data\n """\n headers = {{"Content-Type": "application/json"}}\n \n if method.upper() == "GET":\n response = requests.get(url, headers=headers)\n elif method.upper() == "POST":\n response = requests.post(url, json=data, headers=headers)\n elif method.upper() == "PUT":\n response = requests.put(url, json=data, headers=headers)\n elif method.upper() == "DELETE":\n response = requests.delete(url, headers=headers)\n else:\n raise ValueError(f"Unsupported method: {{method}}")\n \n return {{\n "status_code": response.status_code,\n "headers": dict(response.headers),\n "data": response.json() if response.headers.get("content-type", "").startswith("application/json") else response.text\n }}\n\n\ndef main():\n """Main entry point for the skill."""\n import sys\n import json\n \n if len(sys.argv) > 1:\n result = {func_name}(sys.argv[1])\n print(json.dumps(result, indent=2))\n else:\n print("Usage: python script.py <url>")\n\n\nif __name__ == "__main__":\n main()\n'''
|
||||
|
||||
def _generate_file_script(self, name: str, description: str, func_name: str) -> str:
|
||||
"""Generate file processing script."""
|
||||
return f'''"""\n{name} - {description}\n\nThis script provides file processing functionality.\n"""\nfrom pathlib import Path\n\ndef {func_name}(file_path: str) -> dict:\n """\n Process file and return information.\n \n Args:\n file_path: Path to the file to process\n \n Returns:\n Dictionary with file information and content\n """\n path = Path(file_path)\n \n if not path.exists():\n raise FileNotFoundError(f"File not found: {{file_path}}")\n \n stat = path.stat()\n \n result = {{\n "name": path.name,\n "extension": path.suffix,\n "size_bytes": stat.st_size,\n "is_file": path.is_file(),\n "is_dir": path.is_dir()\n }}\n \n if path.is_file() and stat.st_size < 1024 * 1024:\n try:\n result["content"] = path.read_text()\n result["lines"] = len(result["content"].splitlines())\n except UnicodeDecodeError:\n result["content"] = "<binary file>"\n \n return result\n\n\ndef main():\n """Main entry point for the skill."""\n import sys\n import json\n \n if len(sys.argv) > 1:\n result = {func_name}(sys.argv[1])\n print(json.dumps(result, indent=2))\n else:\n print("Usage: python script.py <file_path>")\n\n\nif __name__ == "__main__":\n main()\n'''
|
||||
|
||||
def _generate_image_script(self, name: str, description: str, func_name: str) -> str:
|
||||
"""Generate image processing script."""
|
||||
return f'''"""\n{name} - {description}\n\nThis script provides image processing functionality.\n"""\nfrom PIL import Image\n\ndef {func_name}(file_path: str, output_path: str = None, resize: tuple = None) -> dict:\n """\n Process image file.\n \n Args:\n file_path: Path to the image file\n output_path: Optional output path for processed image\n resize: Optional tuple (width, height) to resize\n \n Returns:\n Dictionary with image information\n """\n img = Image.open(file_path)\n \n result = {{\n "format": img.format,\n "mode": img.mode,\n "size": img.size,\n "width": img.width,\n "height": img.height\n }}\n \n if resize:\n img = img.resize(resize)\n result["resized_to"] = resize\n \n if output_path:\n img.save(output_path)\n result["saved_to"] = output_path\n \n return result\n\n\ndef main():\n """Main entry point for the skill."""\n import sys\n import json\n \n if len(sys.argv) > 1:\n result = {func_name}(sys.argv[1])\n print(json.dumps(result, indent=2))\n else:\n print("Usage: python script.py <image_file>")\n\n\nif __name__ == "__main__":\n main()\n'''
|
||||
|
||||
def _generate_json_script(self, name: str, description: str, func_name: str) -> str:
|
||||
"""Generate JSON/YAML processing script."""
|
||||
return f'''"""\n{name} - {description}\n\nThis script provides JSON/YAML processing functionality.\n"""\nimport json\nfrom pathlib import Path\n\ndef {func_name}(file_path: str) -> dict:\n """\n Process JSON or YAML file.\n \n Args:\n file_path: Path to the JSON/YAML file\n \n Returns:\n Dictionary with parsed content and metadata\n """\n path = Path(file_path)\n content = path.read_text()\n \n if path.suffix in [".yaml", ".yml"]:\n import yaml\n data = yaml.safe_load(content)\n else:\n data = json.loads(content)\n \n result = {{\n "file": file_path,\n "format": "yaml" if path.suffix in [".yaml", ".yml"] else "json",\n "keys": list(data.keys()) if isinstance(data, dict) else None,\n "length": len(data) if isinstance(data, (list, dict)) else None,\n "data": data\n }}\n \n return result\n\n\ndef main():\n """Main entry point for the skill."""\n import sys\n \n if len(sys.argv) > 1:\n result = {func_name}(sys.argv[1])\n print(json.dumps(result, indent=2))\n else:\n print("Usage: python script.py <json_or_yaml_file>")\n\n\nif __name__ == "__main__":\n main()\n'''
|
||||
|
||||
def _generate_text_script(self, name: str, description: str, func_name: str) -> str:
|
||||
"""Generate text processing script."""
|
||||
return f'''"""\n{name} - {description}\n\nThis script provides text processing functionality.\n"""\nimport re\n\ndef {func_name}(text: str, pattern: str = None, replacement: str = None) -> dict:\n """\n Process text with optional regex operations.\n \n Args:\n text: Input text to process\n pattern: Optional regex pattern to search\n replacement: Optional replacement string\n \n Returns:\n Dictionary with processing results\n """\n result = {{\n "original_length": len(text),\n "lines": len(text.splitlines()),\n "words": len(text.split()),\n "characters": len(text.replace(" ", ""))\n }}\n \n if pattern:\n matches = re.findall(pattern, text)\n result["matches"] = matches\n result["match_count"] = len(matches)\n \n if replacement is not None:\n result["replaced_text"] = re.sub(pattern, replacement, text)\n \n return result\n\n\ndef main():\n """Main entry point for the skill."""\n import sys\n import json\n \n if len(sys.argv) > 1:\n with open(sys.argv[1]) as f:\n text = f.read()\n result = {func_name}(text)\n print(json.dumps(result, indent=2))\n else:\n print("Usage: python script.py <text_file>")\n\n\nif __name__ == "__main__":\n main()\n'''
|
||||
|
||||
def _generate_generic_script(self, name: str, description: str, func_name: str) -> str:
|
||||
"""Generate generic script template."""
|
||||
return f'''"""
|
||||
{name} - {description}
|
||||
|
||||
This script provides functionality for the {name} skill.
|
||||
"""
|
||||
|
||||
def {func_name}(input_data: str) -> str:
|
||||
"""
|
||||
Process input data for {name}.
|
||||
|
||||
Args:
|
||||
input_data: The input to process
|
||||
|
||||
Returns:
|
||||
Processed result
|
||||
"""
|
||||
result = f"Processed: {{input_data}}"
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point for the skill."""
|
||||
import sys
|
||||
if len(sys.argv) > 1:
|
||||
result = {func_name}(sys.argv[1])
|
||||
print(result)
|
||||
else:
|
||||
print("Usage: python script.py <input>")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
'''
|
||||
|
||||
def _generate_skill_content_with_ai(
|
||||
self,
|
||||
name: str,
|
||||
description: str
|
||||
) -> Optional[dict]:
|
||||
"""Generate SKILL.md and optional skill.py using AI.
|
||||
|
||||
Args:
|
||||
name: Skill name
|
||||
description: Skill description/prompt
|
||||
|
||||
Returns:
|
||||
Dict with 'skill_md' and optionally 'skill_py', or None if no API key
|
||||
"""
|
||||
# Check for API keys
|
||||
api_key = (
|
||||
os.environ.get("OPENAI_API_KEY") or
|
||||
os.environ.get("ANTHROPIC_API_KEY") or
|
||||
os.environ.get("GOOGLE_API_KEY") or
|
||||
os.environ.get("GEMINI_API_KEY")
|
||||
)
|
||||
|
||||
if not api_key:
|
||||
return None
|
||||
|
||||
try:
|
||||
from praisonaiagents import Agent
|
||||
|
||||
# Create prompt for AI to generate skill content
|
||||
prompt = f"""Create a comprehensive SKILL.md file for an Agent Skill with the following details:
|
||||
|
||||
Name: {name}
|
||||
Description: {description}
|
||||
|
||||
The SKILL.md must follow this exact format:
|
||||
|
||||
```markdown
|
||||
---
|
||||
name: {name}
|
||||
description: {description}
|
||||
license: Apache-2.0
|
||||
compatibility: Works with PraisonAI Agents, Claude Code, and other Agent Skills compatible tools
|
||||
metadata:
|
||||
author: user
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
# [Title]
|
||||
|
||||
## Overview
|
||||
[Detailed overview of what this skill does]
|
||||
|
||||
## When to Use
|
||||
[Describe when this skill should be activated]
|
||||
|
||||
## Instructions
|
||||
[Step-by-step instructions for the agent]
|
||||
|
||||
## Examples
|
||||
[Provide concrete examples]
|
||||
|
||||
## Best Practices
|
||||
[List best practices]
|
||||
```
|
||||
|
||||
Generate detailed, practical content based on the description. Make it comprehensive but concise.
|
||||
The skill should be immediately useful for an AI agent.
|
||||
|
||||
Also, if this skill requires any Python code to function, provide a script.py that implements the core functionality.
|
||||
The script MUST follow this pattern:
|
||||
1. Have a main function that accepts file path as command line argument (sys.argv[1])
|
||||
2. Print JSON output using json.dumps()
|
||||
3. Include usage message if no arguments provided
|
||||
|
||||
Example script pattern:
|
||||
```python
|
||||
import sys
|
||||
import json
|
||||
|
||||
def process_file(file_path: str) -> dict:
|
||||
# Core logic here
|
||||
return {{"result": "data"}}
|
||||
|
||||
def main():
|
||||
if len(sys.argv) > 1:
|
||||
result = process_file(sys.argv[1])
|
||||
print(json.dumps(result, indent=2))
|
||||
else:
|
||||
print("Usage: python script.py <file_path>")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
```
|
||||
|
||||
Format your response as:
|
||||
---SKILL.MD---
|
||||
[content]
|
||||
---SKILL.PY---
|
||||
[content or "NONE" if no script needed]
|
||||
"""
|
||||
|
||||
agent = Agent(
|
||||
name="SkillGenerator",
|
||||
role="Skill Content Generator",
|
||||
goal="Generate high-quality Agent Skill content",
|
||||
instructions="You are an expert at creating Agent Skills. Generate comprehensive, practical skill content.",
|
||||
llm=os.environ.get("OPENAI_MODEL_NAME", "gpt-4o-mini"),
|
||||
verbose=False
|
||||
)
|
||||
|
||||
result = agent.start(prompt)
|
||||
|
||||
# Parse the response
|
||||
if result and "---SKILL.MD---" in result:
|
||||
parts = result.split("---SKILL.MD---")
|
||||
if len(parts) > 1:
|
||||
skill_md_part = parts[1]
|
||||
skill_py = None
|
||||
|
||||
if "---SKILL.PY---" in skill_md_part:
|
||||
md_parts = skill_md_part.split("---SKILL.PY---")
|
||||
skill_md = self._strip_code_blocks(md_parts[0].strip())
|
||||
skill_py_content = md_parts[1].strip() if len(md_parts) > 1 else None
|
||||
if skill_py_content and skill_py_content.upper() != "NONE":
|
||||
skill_py = self._strip_code_blocks(skill_py_content)
|
||||
else:
|
||||
skill_md = self._strip_code_blocks(skill_md_part.strip())
|
||||
|
||||
return {"skill_md": skill_md, "skill_py": skill_py}
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
if self.verbose:
|
||||
print(f"⚠ AI generation failed: {e}")
|
||||
return None
|
||||
|
||||
def _strip_code_blocks(self, content: str) -> str:
|
||||
"""Strip markdown code block wrappers from content.
|
||||
|
||||
Args:
|
||||
content: Content that may be wrapped in ```markdown or ```python blocks
|
||||
|
||||
Returns:
|
||||
Content with code block wrappers removed
|
||||
"""
|
||||
import re
|
||||
# Match ```language at start and ``` at end
|
||||
pattern = r'^```(?:markdown|python|json|yaml|md)?\s*\n?(.*?)\n?```\s*$'
|
||||
match = re.match(pattern, content, re.DOTALL)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
return content
|
||||
|
||||
def upload_skill(
|
||||
self,
|
||||
skill_path: str,
|
||||
display_title: Optional[str] = None
|
||||
) -> Optional[str]:
|
||||
"""Upload a skill to Anthropic Skills API.
|
||||
|
||||
Args:
|
||||
skill_path: Path to the skill directory
|
||||
display_title: Optional display title for the skill
|
||||
|
||||
Returns:
|
||||
Skill ID if successful, None otherwise
|
||||
"""
|
||||
path = Path(skill_path).expanduser().resolve()
|
||||
|
||||
if not path.exists():
|
||||
raise ValueError(f"Skill path does not exist: {path}")
|
||||
|
||||
skill_md_path = path / "SKILL.md"
|
||||
if not skill_md_path.exists():
|
||||
raise ValueError(f"SKILL.md not found in {path}")
|
||||
|
||||
# Check for Anthropic API key
|
||||
api_key = os.environ.get("ANTHROPIC_API_KEY")
|
||||
if not api_key:
|
||||
print("Error: ANTHROPIC_API_KEY environment variable is required")
|
||||
return None
|
||||
|
||||
try:
|
||||
import anthropic
|
||||
|
||||
client = anthropic.Anthropic(api_key=api_key)
|
||||
|
||||
# Read skill name from SKILL.md
|
||||
skill_md_content = skill_md_path.read_text()
|
||||
skill_name = path.name
|
||||
|
||||
# Extract name from frontmatter if possible
|
||||
if "---" in skill_md_content:
|
||||
import re
|
||||
match = re.search(r'^name:\s*(.+)$', skill_md_content, re.MULTILINE)
|
||||
if match:
|
||||
skill_name = match.group(1).strip()
|
||||
|
||||
title = display_title or skill_name.replace('-', ' ').title()
|
||||
|
||||
if self.verbose:
|
||||
print(f"Uploading skill '{skill_name}' to Anthropic...")
|
||||
|
||||
# Create skill using Anthropic API
|
||||
with open(skill_md_path, 'rb') as f:
|
||||
response = client.beta.skills.create(
|
||||
display_title=title,
|
||||
files=[(f"{skill_name}/SKILL.md", f, "text/markdown")],
|
||||
betas=["skills-2025-10-02"]
|
||||
)
|
||||
|
||||
if self.verbose:
|
||||
print("✓ Skill uploaded successfully!")
|
||||
print(f" ID: {response.id}")
|
||||
print(f" Title: {response.display_title}")
|
||||
|
||||
return response.id
|
||||
|
||||
except ImportError:
|
||||
print("Error: anthropic package is required. Install with: pip install anthropic")
|
||||
return None
|
||||
except Exception as e:
|
||||
if self.verbose:
|
||||
print(f"❌ Upload failed: {e}")
|
||||
return None
|
||||
|
||||
def generate_prompt(
|
||||
self,
|
||||
skill_dirs: Optional[List[str]] = None,
|
||||
include_defaults: bool = True
|
||||
) -> str:
|
||||
"""Generate prompt XML for available skills.
|
||||
|
||||
Args:
|
||||
skill_dirs: Optional list of directories to scan
|
||||
include_defaults: Whether to include default skill directories
|
||||
|
||||
Returns:
|
||||
XML string with <available_skills> block
|
||||
"""
|
||||
from praisonaiagents.skills import SkillManager
|
||||
|
||||
manager = SkillManager()
|
||||
|
||||
if skill_dirs:
|
||||
manager.discover(skill_dirs, include_defaults=include_defaults)
|
||||
elif include_defaults:
|
||||
manager.discover(include_defaults=True)
|
||||
|
||||
prompt = manager.to_prompt()
|
||||
|
||||
if self.verbose:
|
||||
print(prompt)
|
||||
|
||||
return prompt
|
||||
|
||||
|
||||
def handle_skills_command(args) -> int:
|
||||
"""Handle skills subcommand from CLI.
|
||||
|
||||
Args:
|
||||
args: Parsed command line arguments
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success, non-zero for error)
|
||||
"""
|
||||
handler = SkillsHandler(verbose=True)
|
||||
|
||||
try:
|
||||
if args.skills_command == "list":
|
||||
dirs = args.dirs if hasattr(args, 'dirs') and args.dirs else None
|
||||
handler.list_skills(dirs, include_defaults=True)
|
||||
|
||||
elif args.skills_command == "validate":
|
||||
if not hasattr(args, 'path') or not args.path:
|
||||
print("Error: --path is required for validate command")
|
||||
return 1
|
||||
result = handler.validate_skill(args.path)
|
||||
return 0 if result["valid"] else 1
|
||||
|
||||
elif args.skills_command == "create":
|
||||
if not hasattr(args, 'name') or not args.name:
|
||||
print("Error: --name is required for create command")
|
||||
return 1
|
||||
|
||||
# Get all optional arguments
|
||||
description = getattr(args, 'description', None) or "A custom skill"
|
||||
output_dir = getattr(args, 'output_dir', None) or getattr(args, 'output', None)
|
||||
author = getattr(args, 'author', None)
|
||||
license_type = getattr(args, 'license', None)
|
||||
compatibility = getattr(args, 'compatibility', None)
|
||||
template = getattr(args, 'template', False)
|
||||
generate_script = getattr(args, 'script', False)
|
||||
|
||||
handler.create_skill(
|
||||
name=args.name,
|
||||
description=description,
|
||||
output_dir=output_dir,
|
||||
author=author,
|
||||
license=license_type,
|
||||
compatibility=compatibility,
|
||||
template=template,
|
||||
use_ai=not template,
|
||||
generate_script=generate_script
|
||||
)
|
||||
|
||||
elif args.skills_command == "prompt":
|
||||
dirs = args.dirs if hasattr(args, 'dirs') and args.dirs else None
|
||||
handler.generate_prompt(dirs, include_defaults=True)
|
||||
|
||||
elif args.skills_command == "upload":
|
||||
if not hasattr(args, 'path') or not args.path:
|
||||
print("Error: --path is required for upload command")
|
||||
return 1
|
||||
title = getattr(args, 'title', None)
|
||||
handler.upload_skill(args.path, title)
|
||||
|
||||
else:
|
||||
print(f"Unknown skills command: {args.skills_command}")
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def add_skills_parser(subparsers) -> None:
|
||||
"""Add skills subcommand to argument parser.
|
||||
|
||||
Args:
|
||||
subparsers: Subparsers object from argparse
|
||||
"""
|
||||
# subparsers is already the skills subparsers object
|
||||
# We just need to add the subcommands to it
|
||||
|
||||
# list command
|
||||
list_parser = subparsers.add_parser(
|
||||
'list',
|
||||
help='List available skills'
|
||||
)
|
||||
list_parser.add_argument(
|
||||
'--dirs',
|
||||
nargs='+',
|
||||
help='Directories to scan for skills'
|
||||
)
|
||||
|
||||
# validate command
|
||||
validate_parser = subparsers.add_parser(
|
||||
'validate',
|
||||
help='Validate a skill directory'
|
||||
)
|
||||
validate_parser.add_argument(
|
||||
'--path',
|
||||
required=True,
|
||||
help='Path to skill directory'
|
||||
)
|
||||
|
||||
# create command
|
||||
create_parser = subparsers.add_parser(
|
||||
'create',
|
||||
help='Create a new skill (uses AI by default, --template for template only)'
|
||||
)
|
||||
create_parser.add_argument(
|
||||
'--name',
|
||||
required=True,
|
||||
help='Skill name (kebab-case, e.g., my-skill)'
|
||||
)
|
||||
create_parser.add_argument(
|
||||
'--description',
|
||||
default='A custom skill',
|
||||
help='Skill description (used as prompt for AI generation)'
|
||||
)
|
||||
create_parser.add_argument(
|
||||
'--output-dir', '--output',
|
||||
dest='output_dir',
|
||||
help='Output directory (default: current directory)'
|
||||
)
|
||||
create_parser.add_argument(
|
||||
'--author',
|
||||
help='Author name for skill metadata'
|
||||
)
|
||||
create_parser.add_argument(
|
||||
'--license',
|
||||
help='License type (default: Apache-2.0)'
|
||||
)
|
||||
create_parser.add_argument(
|
||||
'--compatibility',
|
||||
help='Compatibility information'
|
||||
)
|
||||
create_parser.add_argument(
|
||||
'--template',
|
||||
action='store_true',
|
||||
help='Use template only, skip AI generation'
|
||||
)
|
||||
create_parser.add_argument(
|
||||
'--script',
|
||||
action='store_true',
|
||||
help='Generate scripts/skill.py with template code'
|
||||
)
|
||||
|
||||
# prompt command
|
||||
prompt_parser = subparsers.add_parser(
|
||||
'prompt',
|
||||
help='Generate prompt XML for skills'
|
||||
)
|
||||
prompt_parser.add_argument(
|
||||
'--dirs',
|
||||
nargs='+',
|
||||
help='Directories to scan for skills'
|
||||
)
|
||||
|
||||
# upload command - upload skill to Anthropic
|
||||
upload_parser = subparsers.add_parser(
|
||||
'upload',
|
||||
help='Upload skill to Anthropic Skills API'
|
||||
)
|
||||
upload_parser.add_argument(
|
||||
'--path',
|
||||
required=True,
|
||||
help='Path to skill directory'
|
||||
)
|
||||
upload_parser.add_argument(
|
||||
'--title',
|
||||
help='Display title for the skill'
|
||||
)
|
||||
@@ -5,6 +5,42 @@ import argparse
|
||||
import warnings
|
||||
import os
|
||||
|
||||
# Suppress Pydantic serialization warnings from LiteLLM BEFORE any imports
|
||||
# These warnings occur when LiteLLM's response objects have field mismatches
|
||||
# Using both filterwarnings AND patching warnings.warn for complete suppression
|
||||
|
||||
warnings.filterwarnings("ignore", message=".*Pydantic serializer warnings.*")
|
||||
warnings.filterwarnings("ignore", message=".*PydanticSerializationUnexpectedValue.*")
|
||||
warnings.filterwarnings("ignore", message=".*Expected \\d+ fields but got.*")
|
||||
warnings.filterwarnings("ignore", message=".*Expected `StreamingChoices`.*")
|
||||
warnings.filterwarnings("ignore", message=".*Expected `Message`.*")
|
||||
warnings.filterwarnings("ignore", message=".*serialized value may not be as expected.*")
|
||||
warnings.filterwarnings("ignore", category=UserWarning, module="pydantic.*")
|
||||
|
||||
# Patch warnings.showwarning to intercept ALL warnings including those from crewai's patched warn
|
||||
# This is the final output function that actually displays warnings
|
||||
_SUPPRESSED_PATTERNS = [
|
||||
"Pydantic serializer warnings",
|
||||
"PydanticSerializationUnexpectedValue",
|
||||
"Expected", # Catches "Expected N fields but got M"
|
||||
"StreamingChoices",
|
||||
"serialized value may not be as expected",
|
||||
"duckduckgo_search", # Suppress duckduckgo rename warning
|
||||
]
|
||||
|
||||
_original_showwarning = warnings.showwarning
|
||||
|
||||
def _patched_showwarning(message, category, filename, lineno, file=None, line=None):
|
||||
msg_str = str(message)
|
||||
for pattern in _SUPPRESSED_PATTERNS:
|
||||
if pattern in msg_str:
|
||||
return
|
||||
if category is UserWarning and "pydantic" in filename.lower():
|
||||
return
|
||||
_original_showwarning(message, category, filename, lineno, file, line)
|
||||
|
||||
warnings.showwarning = _patched_showwarning
|
||||
|
||||
# Suppress crewai RuntimeWarning about module loading order (only in non-debug mode)
|
||||
# This warning is harmless and occurs when running as `python -m praisonai.cli.main`
|
||||
if os.environ.get('LOGLEVEL', 'INFO').upper() != 'DEBUG':
|
||||
@@ -59,49 +95,27 @@ def _get_chainlit_run():
|
||||
from chainlit.cli import chainlit_run
|
||||
return chainlit_run
|
||||
|
||||
try:
|
||||
import gradio as gr
|
||||
GRADIO_AVAILABLE = True
|
||||
except ImportError:
|
||||
pass
|
||||
# Use find_spec for fast availability checks (no actual import)
|
||||
import importlib.util
|
||||
GRADIO_AVAILABLE = importlib.util.find_spec("gradio") is not None
|
||||
CALL_MODULE_AVAILABLE = importlib.util.find_spec("praisonai.api.call") is not None
|
||||
CREWAI_AVAILABLE = importlib.util.find_spec("crewai") is not None
|
||||
AUTOGEN_AVAILABLE = importlib.util.find_spec("autogen") is not None
|
||||
PRAISONAI_AVAILABLE = importlib.util.find_spec("praisonaiagents") is not None
|
||||
TRAIN_AVAILABLE = importlib.util.find_spec("unsloth") is not None
|
||||
|
||||
try:
|
||||
import praisonai.api.call as call_module
|
||||
CALL_MODULE_AVAILABLE = True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
import crewai
|
||||
CREWAI_AVAILABLE = True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
import autogen
|
||||
AUTOGEN_AVAILABLE = True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from praisonaiagents import Agent as PraisonAgent, Task as PraisonTask, PraisonAIAgents
|
||||
PRAISONAI_AVAILABLE = True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from unsloth import FastLanguageModel
|
||||
TRAIN_AVAILABLE = True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO'), format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logging.basicConfig(level=os.environ.get('LOGLEVEL', 'WARNING'), format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logging.getLogger('alembic').setLevel(logging.ERROR)
|
||||
logging.getLogger('gradio').setLevel(logging.ERROR)
|
||||
logging.getLogger('gradio').setLevel(os.environ.get('GRADIO_LOGLEVEL', 'WARNING'))
|
||||
logging.getLogger('rust_logger').setLevel(logging.WARNING)
|
||||
logging.getLogger('duckduckgo').setLevel(logging.ERROR)
|
||||
logging.getLogger('_client').setLevel(logging.ERROR)
|
||||
# Suppress praisonaiagents INFO logs unless LOGLEVEL is explicitly set to debug/info
|
||||
if os.environ.get('LOGLEVEL', '').upper() not in ('DEBUG', 'INFO'):
|
||||
logging.getLogger('praisonaiagents').setLevel(logging.WARNING)
|
||||
logging.getLogger('praisonaiagents.llm').setLevel(logging.WARNING)
|
||||
logging.getLogger('praisonaiagents.llm.llm').setLevel(logging.WARNING)
|
||||
|
||||
def stream_subprocess(command, env=None):
|
||||
"""
|
||||
@@ -215,7 +229,14 @@ class PraisonAI:
|
||||
# Store the original agent_file from constructor
|
||||
original_agent_file = self.agent_file
|
||||
|
||||
args = self.parse_args()
|
||||
# Parse args - this returns both args and unknown_args
|
||||
parse_result = self.parse_args()
|
||||
if isinstance(parse_result, tuple):
|
||||
args, unknown_args = parse_result
|
||||
else:
|
||||
args = parse_result
|
||||
unknown_args = []
|
||||
|
||||
# Store args for use in handle_direct_prompt
|
||||
self.args = args
|
||||
invocation_cmd = "praisonai"
|
||||
@@ -245,10 +266,36 @@ class PraisonAI:
|
||||
file_input = self.read_file_if_provided(getattr(args, 'file', None))
|
||||
|
||||
if args.command:
|
||||
if args.command.startswith("tests.test") or args.command.startswith("tests/test"): # Argument used for testing purposes
|
||||
# Handle schedule command
|
||||
if args.command == "schedule":
|
||||
from praisonai.cli.features.agent_scheduler import AgentSchedulerHandler
|
||||
# Check for subcommands (start, list, stop, logs, restart)
|
||||
subcommand = unknown_args[0] if unknown_args and not unknown_args[0].startswith('-') else None
|
||||
|
||||
if subcommand in ['start', 'list', 'stop', 'logs', 'restart', 'delete', 'describe', 'save', 'stop-all', 'stats']:
|
||||
exit_code = AgentSchedulerHandler.handle_daemon_command(subcommand, args, unknown_args[1:] if len(unknown_args) > 1 else [])
|
||||
else:
|
||||
# Legacy mode: direct scheduling (foreground) or daemon mode
|
||||
daemon_mode = getattr(args, 'daemon', False)
|
||||
exit_code = AgentSchedulerHandler.handle_schedule_command(args, unknown_args, daemon_mode=daemon_mode)
|
||||
|
||||
sys.exit(exit_code)
|
||||
elif args.command.startswith("tests.test") or args.command.startswith("tests/test"): # Argument used for testing purposes
|
||||
print("test")
|
||||
return "test"
|
||||
else:
|
||||
# Handle --compare flag for CLI mode comparison
|
||||
if hasattr(args, 'compare') and args.compare:
|
||||
from .features.compare import CompareHandler
|
||||
handler = CompareHandler(verbose=getattr(args, 'verbose', False))
|
||||
result = handler.execute(
|
||||
args.command,
|
||||
args.compare,
|
||||
model=getattr(args, 'llm', None),
|
||||
output_path=getattr(args, 'compare_output', None)
|
||||
)
|
||||
return result
|
||||
|
||||
# Combine command with any available inputs (stdin and/or file)
|
||||
combined_inputs = []
|
||||
if stdin_input:
|
||||
@@ -266,6 +313,18 @@ class PraisonAI:
|
||||
elif hasattr(args, 'direct_prompt') and args.direct_prompt:
|
||||
# Only handle direct prompt if agent_file wasn't explicitly set in constructor
|
||||
if original_agent_file == "agents.yaml": # Default value, so safe to use direct prompt
|
||||
# Handle --compare flag for CLI mode comparison
|
||||
if hasattr(args, 'compare') and args.compare:
|
||||
from .features.compare import CompareHandler
|
||||
handler = CompareHandler(verbose=getattr(args, 'verbose', False))
|
||||
result = handler.execute(
|
||||
args.direct_prompt,
|
||||
args.compare,
|
||||
model=getattr(args, 'llm', None),
|
||||
output_path=getattr(args, 'compare_output', None)
|
||||
)
|
||||
return result
|
||||
|
||||
# Combine direct prompt with any available inputs (stdin and/or file)
|
||||
prompt_parts = [args.direct_prompt]
|
||||
if stdin_input:
|
||||
@@ -599,10 +658,11 @@ class PraisonAI:
|
||||
default_args.realtime = False
|
||||
default_args.call = False
|
||||
default_args.public = False
|
||||
default_args.chat_mode = False
|
||||
return default_args
|
||||
|
||||
# Define special commands
|
||||
special_commands = ['chat', 'code', 'call', 'realtime', 'train', 'ui', 'context', 'research', 'memory', 'rules', 'workflow', 'hooks', 'knowledge', 'session', 'tools', 'todo', 'docs', 'mcp', 'commit', 'serve']
|
||||
special_commands = ['chat', 'code', 'call', 'realtime', 'train', 'ui', 'context', 'research', 'memory', 'rules', 'workflow', 'hooks', 'knowledge', 'session', 'tools', 'todo', 'docs', 'mcp', 'commit', 'serve', 'schedule', 'skills', 'profile', 'eval', 'agents']
|
||||
|
||||
parser = argparse.ArgumentParser(prog="praisonai", description="praisonAI command-line interface")
|
||||
parser.add_argument("--framework", choices=["crewai", "autogen", "praisonai"], help="Specify the framework")
|
||||
@@ -674,8 +734,12 @@ class PraisonAI:
|
||||
# Metrics - token/cost tracking
|
||||
parser.add_argument("--metrics", action="store_true", help="Display token usage and cost metrics")
|
||||
|
||||
# Image - vision processing
|
||||
parser.add_argument("--image", type=str, help="Path to image file for vision-based tasks")
|
||||
# Image Description (Vision) - analyze existing images
|
||||
parser.add_argument("--image", type=str, help="Path to image file for vision-based description/analysis")
|
||||
|
||||
# Image Generation - create new images from text
|
||||
parser.add_argument("--image-generate", action="store_true", dest="image_generate",
|
||||
help="Generate an image from the text prompt (use with --llm for model selection)")
|
||||
|
||||
# Telemetry - usage monitoring
|
||||
parser.add_argument("--telemetry", action="store_true", help="Enable usage monitoring and analytics")
|
||||
@@ -720,9 +784,29 @@ class PraisonAI:
|
||||
# Autonomy Mode - control AI action approval
|
||||
parser.add_argument("--autonomy", type=str, choices=["suggest", "auto_edit", "full_auto"], help="Set autonomy mode for AI actions")
|
||||
|
||||
# Tool Approval - control tool execution approval
|
||||
parser.add_argument("--trust", action="store_true", help="Auto-approve all tool executions (skip approval prompts)")
|
||||
parser.add_argument("--approve-level", type=str, choices=["low", "medium", "high", "critical"],
|
||||
help="Auto-approve tools up to this risk level (e.g., --approve-level high approves low/medium/high but prompts for critical)")
|
||||
|
||||
# Sandbox Execution - secure command execution
|
||||
parser.add_argument("--sandbox", type=str, choices=["off", "basic", "strict"], help="Enable sandboxed command execution")
|
||||
|
||||
# External Agent - use external AI CLI tools
|
||||
parser.add_argument("--external-agent", type=str, choices=["claude", "gemini", "codex", "cursor"],
|
||||
help="Use external AI CLI tool (claude, gemini, codex, cursor)")
|
||||
|
||||
# Compare - compare different CLI modes
|
||||
parser.add_argument("--compare", type=str, help="Compare CLI modes (comma-separated: basic,tools,research,planning)")
|
||||
parser.add_argument("--compare-output", type=str, help="Save comparison results to file")
|
||||
|
||||
# Agent Scheduler - for schedule command
|
||||
parser.add_argument("--interval", dest="schedule_interval", type=str, help="Schedule interval (e.g., 'hourly', '*/30m', 'daily')")
|
||||
parser.add_argument("--schedule-max-retries", dest="schedule_max_retries", type=int, help="Maximum retry attempts for scheduled execution")
|
||||
parser.add_argument("--timeout", type=int, help="Maximum execution time per run in seconds")
|
||||
parser.add_argument('--max-cost', type=float, help='Maximum total cost budget in USD')
|
||||
parser.add_argument('--daemon', action='store_true', help=argparse.SUPPRESS) # Hidden flag for daemon mode
|
||||
|
||||
# If we're in a test environment, parse with empty args to avoid pytest interference
|
||||
if in_test_env:
|
||||
args, unknown_args = parser.parse_known_args([])
|
||||
@@ -783,7 +867,7 @@ class PraisonAI:
|
||||
except ModuleNotFoundError as e:
|
||||
missing_module = str(e).split("'")[1]
|
||||
print(f"[red]ERROR: Missing dependency {missing_module}. Install with:[/red]")
|
||||
print(f"\npip install \"praisonai[chat]\"\n")
|
||||
print("\npip install \"praisonai[chat]\"\n")
|
||||
sys.exit(1)
|
||||
sys.exit(0)
|
||||
|
||||
@@ -1011,6 +1095,53 @@ class PraisonAI:
|
||||
self.handle_commit_command(unknown_args)
|
||||
sys.exit(0)
|
||||
|
||||
elif args.command == 'skills':
|
||||
if not PRAISONAI_AVAILABLE:
|
||||
print("[red]ERROR: PraisonAI Agents is not installed. Install with:[/red]")
|
||||
print("\npip install praisonaiagents\n")
|
||||
sys.exit(1)
|
||||
|
||||
from .features.skills import handle_skills_command, add_skills_parser
|
||||
|
||||
# Create a parser for skills command
|
||||
skills_parser = argparse.ArgumentParser(prog="praisonai skills")
|
||||
skills_subparsers = skills_parser.add_subparsers(dest='skills_command', help='Skills commands')
|
||||
add_skills_parser(skills_subparsers)
|
||||
skills_args = skills_parser.parse_args(unknown_args)
|
||||
|
||||
exit_code = handle_skills_command(skills_args)
|
||||
sys.exit(exit_code)
|
||||
|
||||
elif args.command == 'profile':
|
||||
# Profile command - performance profiling and benchmarking
|
||||
from .features.profile import handle_profile_command
|
||||
exit_code = handle_profile_command(unknown_args)
|
||||
sys.exit(exit_code)
|
||||
|
||||
elif args.command == 'eval':
|
||||
# Eval command - evaluate model responses against expected outputs
|
||||
from .features.eval import handle_eval_command
|
||||
exit_code = handle_eval_command(unknown_args)
|
||||
sys.exit(exit_code)
|
||||
|
||||
elif args.command == 'agents':
|
||||
# Agents command - run multiple agents with custom definitions
|
||||
if not PRAISONAI_AVAILABLE:
|
||||
print("[red]ERROR: PraisonAI Agents is not installed. Install with:[/red]")
|
||||
print("\npip install praisonaiagents\n")
|
||||
sys.exit(1)
|
||||
|
||||
from .features.agents import handle_agents_command, add_agents_parser
|
||||
|
||||
# Create a parser for agents command
|
||||
agents_parser = argparse.ArgumentParser(prog="praisonai agents")
|
||||
agents_subparsers = agents_parser.add_subparsers(dest='agents_command', help='Agents commands')
|
||||
add_agents_parser(agents_subparsers)
|
||||
agents_args = agents_parser.parse_args(unknown_args)
|
||||
|
||||
exit_code = handle_agents_command(agents_args)
|
||||
sys.exit(exit_code)
|
||||
|
||||
# Only check framework availability for agent-related operations
|
||||
if not args.command and (args.init or args.auto or args.framework):
|
||||
if not CREWAI_AVAILABLE and not AUTOGEN_AVAILABLE and not PRAISONAI_AVAILABLE:
|
||||
@@ -1027,7 +1158,7 @@ class PraisonAI:
|
||||
args.direct_prompt = args.command
|
||||
args.command = None
|
||||
|
||||
return args
|
||||
return args, unknown_args
|
||||
|
||||
def _rewrite_query(self, query: str, rewrite_tools: str = None, verbose: bool = False) -> str:
|
||||
"""
|
||||
@@ -1294,12 +1425,47 @@ class PraisonAI:
|
||||
|
||||
console.print(table)
|
||||
|
||||
# Show recent memories
|
||||
# Show recent short-term memories
|
||||
print("\n[bold]Recent Short-term Memories:[/bold]")
|
||||
short_term = memory.get_short_term(limit=5)
|
||||
for i, item in enumerate(short_term, 1):
|
||||
content = item.get('content', str(item))[:100]
|
||||
print(f" {i}. {content}")
|
||||
if short_term:
|
||||
for i, item in enumerate(short_term, 1):
|
||||
content = item.get('content', str(item))[:100]
|
||||
print(f" {i}. {content}")
|
||||
else:
|
||||
print(" [dim]No short-term memories[/dim]")
|
||||
|
||||
# Show long-term memories
|
||||
print("\n[bold]Long-term Memories:[/bold]")
|
||||
long_term = memory.get_long_term(limit=10)
|
||||
if long_term:
|
||||
for i, item in enumerate(long_term, 1):
|
||||
# Handle both dict and MemoryItem objects
|
||||
if hasattr(item, 'content'):
|
||||
content = str(item.content)[:100]
|
||||
importance = getattr(item, 'importance', 0)
|
||||
else:
|
||||
content = item.get('content', str(item))[:100]
|
||||
importance = item.get('importance', 0)
|
||||
print(f" {i}. [{importance:.1f}] {content}")
|
||||
else:
|
||||
print(" [dim]No long-term memories[/dim]")
|
||||
|
||||
# Show entities
|
||||
print("\n[bold]Entities:[/bold]")
|
||||
entities = memory.get_all_entities()
|
||||
if entities:
|
||||
for entity in entities[:10]:
|
||||
# Handle both dict and Entity objects
|
||||
if hasattr(entity, 'name'):
|
||||
name = entity.name
|
||||
entity_type = getattr(entity, 'entity_type', 'unknown')
|
||||
else:
|
||||
name = entity.get('name', 'Unknown')
|
||||
entity_type = entity.get('entity_type', 'unknown')
|
||||
print(f" • {name} ({entity_type})")
|
||||
else:
|
||||
print(" [dim]No entities[/dim]")
|
||||
|
||||
elif action == 'add':
|
||||
if not action_args:
|
||||
@@ -1532,6 +1698,7 @@ class PraisonAI:
|
||||
args: Parsed command line arguments
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents import Agent as PraisonAgent
|
||||
from praisonaiagents.memory import WorkflowManager
|
||||
from rich import print
|
||||
from rich.table import Table
|
||||
@@ -2773,6 +2940,7 @@ Provide ONLY the commit message, no explanations."""
|
||||
Returns:
|
||||
Final output from the workflow
|
||||
"""
|
||||
from praisonaiagents import Agent as PraisonAgent
|
||||
from rich import print
|
||||
|
||||
workflow_str = self.args.workflow
|
||||
@@ -2919,6 +3087,8 @@ Provide ONLY the commit message, no explanations."""
|
||||
prompt = f"{mention_context}# Task:\n{prompt}"
|
||||
|
||||
if PRAISONAI_AVAILABLE:
|
||||
from praisonaiagents import Agent as PraisonAgent
|
||||
|
||||
agent_config = {
|
||||
"name": "DirectAgent",
|
||||
"role": "Assistant",
|
||||
@@ -2926,10 +3096,18 @@ Provide ONLY the commit message, no explanations."""
|
||||
"backstory": "You are a helpful AI assistant"
|
||||
}
|
||||
|
||||
# Suppress verbose output in interactive mode (no Agent Info, Task, Response panels)
|
||||
if getattr(self, '_interactive_mode', False):
|
||||
# Set verbose=False by default, only enable with --verbose flag
|
||||
# This shows minimal "Generating..." status instead of full Agent Info panels
|
||||
if hasattr(self, 'args') and getattr(self.args, 'verbose', False):
|
||||
agent_config["verbose"] = True
|
||||
else:
|
||||
agent_config["verbose"] = False
|
||||
|
||||
# Load default tools (same as interactive mode)
|
||||
default_tools = self._load_interactive_tools()
|
||||
if default_tools:
|
||||
agent_config["tools"] = default_tools
|
||||
|
||||
# Add llm if specified
|
||||
if hasattr(self, 'args') and self.args.llm:
|
||||
# Check if max_tokens is specified - pass as dict config
|
||||
@@ -2937,7 +3115,7 @@ Provide ONLY the commit message, no explanations."""
|
||||
if max_tokens:
|
||||
# Pass llm as dict with model and max_tokens
|
||||
agent_config["llm"] = {"model": self.args.llm, "max_tokens": max_tokens}
|
||||
print(f"[bold cyan]Max tokens set to: {max_tokens}[/bold cyan]")
|
||||
logging.debug(f"Max tokens set to: {max_tokens}")
|
||||
else:
|
||||
agent_config["llm"] = self.args.llm
|
||||
|
||||
@@ -3010,6 +3188,35 @@ Provide ONLY the commit message, no explanations."""
|
||||
|
||||
# ===== NEW CLI FEATURES INTEGRATION =====
|
||||
|
||||
# Tool Approval - Auto-approve tools based on --trust or --approve-level
|
||||
if getattr(self.args, 'trust', False) or getattr(self.args, 'approve_level', None):
|
||||
from praisonaiagents.approval import set_approval_callback, ApprovalDecision
|
||||
|
||||
if getattr(self.args, 'trust', False):
|
||||
# Auto-approve all tools
|
||||
def auto_approve_all(function_name, arguments, risk_level):
|
||||
return ApprovalDecision(approved=True, reason="Auto-approved via --trust flag")
|
||||
set_approval_callback(auto_approve_all)
|
||||
print("[bold yellow]⚠️ Trust mode enabled - all tool executions will be auto-approved[/bold yellow]")
|
||||
elif getattr(self.args, 'approve_level', None):
|
||||
# Auto-approve up to specified risk level
|
||||
max_level = self.args.approve_level
|
||||
risk_order = {"low": 1, "medium": 2, "high": 3, "critical": 4}
|
||||
max_level_value = risk_order.get(max_level, 3)
|
||||
|
||||
def level_based_approve(function_name, arguments, risk_level):
|
||||
tool_level_value = risk_order.get(risk_level, 4)
|
||||
if tool_level_value <= max_level_value:
|
||||
return ApprovalDecision(approved=True, reason=f"Auto-approved (level {risk_level} <= {max_level})")
|
||||
else:
|
||||
# Signal to pause Live display before showing approval prompt
|
||||
# This is handled by the approval_pending flag in status_info
|
||||
from praisonaiagents.approval import console_approval_callback
|
||||
return console_approval_callback(function_name, arguments, risk_level)
|
||||
|
||||
set_approval_callback(level_based_approve)
|
||||
print(f"[bold cyan]Auto-approve enabled for tools up to '{max_level}' risk level[/bold cyan]")
|
||||
|
||||
# Router - Smart model selection (must be before agent creation)
|
||||
if getattr(self.args, 'router', False):
|
||||
from .features.router import RouterHandler
|
||||
@@ -3018,10 +3225,9 @@ Provide ONLY the commit message, no explanations."""
|
||||
selected_model = router.select_model(prompt, provider)
|
||||
agent_config["llm"] = selected_model
|
||||
|
||||
# Metrics - Token/cost tracking
|
||||
# Metrics - Token/cost tracking (display happens AFTER execution)
|
||||
if getattr(self.args, 'metrics', False):
|
||||
agent_config["metrics"] = True
|
||||
print("[bold cyan]Metrics enabled - will display token usage and costs[/bold cyan]")
|
||||
|
||||
# Telemetry - Usage monitoring
|
||||
if getattr(self.args, 'telemetry', False):
|
||||
@@ -3029,9 +3235,14 @@ Provide ONLY the commit message, no explanations."""
|
||||
telemetry = TelemetryHandler(verbose=getattr(self.args, 'verbose', False))
|
||||
telemetry.enable()
|
||||
|
||||
# Auto Memory - Automatic memory extraction
|
||||
# Sandbox - Secure command execution (display only, actual sandboxing is in tool approval)
|
||||
sandbox_mode = getattr(self.args, 'sandbox', None)
|
||||
if sandbox_mode and sandbox_mode != 'off':
|
||||
print(f"[bold green]🔒 Sandbox Mode: {sandbox_mode.upper()}[/bold green]")
|
||||
print("[dim]Commands will be validated before execution[/dim]")
|
||||
|
||||
# Auto Memory - Automatic memory extraction (handled post-processing, not as Agent param)
|
||||
if getattr(self.args, 'auto_memory', False):
|
||||
agent_config["auto_memory"] = True
|
||||
print("[bold cyan]Auto Memory enabled - will extract and store memories[/bold cyan]")
|
||||
|
||||
# MCP - Model Context Protocol tools
|
||||
@@ -3050,6 +3261,43 @@ Provide ONLY the commit message, no explanations."""
|
||||
existing_tools = list(mcp_tools)
|
||||
agent_config['tools'] = existing_tools
|
||||
|
||||
# External Agent - Use external AI CLI tools directly
|
||||
if getattr(self.args, 'external_agent', None):
|
||||
from rich.console import Console
|
||||
ext_console = Console()
|
||||
external_agent_name = self.args.external_agent
|
||||
try:
|
||||
from .features.external_agents import ExternalAgentsHandler
|
||||
handler = ExternalAgentsHandler(verbose=getattr(self.args, 'verbose', False))
|
||||
|
||||
# Get workspace from current directory
|
||||
import os
|
||||
workspace = os.getcwd()
|
||||
|
||||
integration = handler.get_integration(external_agent_name, workspace=workspace)
|
||||
|
||||
if integration.is_available:
|
||||
ext_console.print(f"[bold cyan]🔌 Using external agent: {external_agent_name}[/bold cyan]")
|
||||
|
||||
# Run the external agent directly instead of PraisonAI agent
|
||||
import asyncio
|
||||
try:
|
||||
result = asyncio.run(integration.execute(prompt))
|
||||
ext_console.print(f"\n[bold green]Result from {external_agent_name}:[/bold green]")
|
||||
ext_console.print(result)
|
||||
# Return empty string to avoid duplicate printing by caller
|
||||
return ""
|
||||
except Exception as e:
|
||||
ext_console.print(f"[red]Error executing {external_agent_name}: {e}[/red]")
|
||||
return None
|
||||
else:
|
||||
ext_console.print(f"[yellow]⚠️ External agent '{external_agent_name}' is not installed[/yellow]")
|
||||
ext_console.print(f"[dim]Install with: {handler._get_install_instructions(external_agent_name)}[/dim]")
|
||||
return None
|
||||
except Exception as e:
|
||||
ext_console.print(f"[red]Error setting up external agent: {e}[/red]")
|
||||
return None
|
||||
|
||||
# Fast Context - Codebase search
|
||||
if getattr(self.args, 'fast_context', None):
|
||||
from .features.fast_context import FastContextHandler
|
||||
@@ -3083,7 +3331,7 @@ Provide ONLY the commit message, no explanations."""
|
||||
|
||||
return result
|
||||
|
||||
# Image processing - Use ImageAgent instead
|
||||
# Image Description (Vision) - analyze existing images
|
||||
if hasattr(self, 'args') and getattr(self.args, 'image', None):
|
||||
from .features.image import ImageHandler
|
||||
image_handler = ImageHandler(verbose=getattr(self.args, 'verbose', False))
|
||||
@@ -3105,6 +3353,27 @@ Provide ONLY the commit message, no explanations."""
|
||||
|
||||
return result
|
||||
|
||||
# Image Generation - create new images from text
|
||||
if hasattr(self, 'args') and getattr(self.args, 'image_generate', False):
|
||||
from .features.image import ImageGenerateHandler
|
||||
image_gen_handler = ImageGenerateHandler(verbose=getattr(self.args, 'verbose', False))
|
||||
result = image_gen_handler.execute(
|
||||
prompt=prompt,
|
||||
llm=agent_config.get('llm')
|
||||
)
|
||||
|
||||
# Format output for display
|
||||
if isinstance(result, dict):
|
||||
if 'error' in result:
|
||||
print(f"[red]Error: {result['error']}[/red]")
|
||||
elif 'data' in result and len(result['data']) > 0:
|
||||
image_url = result['data'][0].get('url', result['data'][0].get('b64_json', 'Generated'))
|
||||
print(f"[green]Image generated successfully![/green]")
|
||||
if 'url' in result['data'][0]:
|
||||
print(f"URL: {result['data'][0]['url']}")
|
||||
|
||||
return result
|
||||
|
||||
# Flow Display - Visual workflow tracking
|
||||
if hasattr(self, 'args') and getattr(self.args, 'flow_display', False):
|
||||
from .features.flow_display import FlowDisplayHandler
|
||||
@@ -3112,7 +3381,13 @@ Provide ONLY the commit message, no explanations."""
|
||||
flow.display_workflow_start("Direct Prompt", ["DirectAgent"])
|
||||
|
||||
agent = PraisonAgent(**agent_config)
|
||||
result = agent.start(prompt)
|
||||
|
||||
# Run with minimal status display when verbose=False
|
||||
is_verbose = agent_config.get("verbose", False)
|
||||
if not is_verbose:
|
||||
result = self._run_with_status_display(agent, prompt)
|
||||
else:
|
||||
result = agent.start(prompt)
|
||||
|
||||
# ===== POST-PROCESSING WITH NEW FEATURES =====
|
||||
|
||||
@@ -3134,7 +3409,10 @@ Provide ONLY the commit message, no explanations."""
|
||||
if hasattr(self, 'args') and getattr(self.args, 'auto_memory', False):
|
||||
from .features.auto_memory import AutoMemoryHandler
|
||||
auto_mem = AutoMemoryHandler(verbose=getattr(self.args, 'verbose', False))
|
||||
auto_mem.post_process_result(result, {'user_id': getattr(self.args, 'user_id', None)})
|
||||
auto_mem.post_process_result(
|
||||
result,
|
||||
{'user_id': getattr(self.args, 'user_id', None), 'user_message': prompt}
|
||||
)
|
||||
|
||||
# Todo - Generate todo list from response
|
||||
if hasattr(self, 'args') and getattr(self.args, 'todo', False):
|
||||
@@ -3142,6 +3420,12 @@ Provide ONLY the commit message, no explanations."""
|
||||
todo = TodoHandler(verbose=getattr(self.args, 'verbose', False))
|
||||
todo.post_process_result(result, True)
|
||||
|
||||
# Telemetry - Display usage summary after execution
|
||||
if hasattr(self, 'args') and getattr(self.args, 'telemetry', False):
|
||||
from .features.telemetry import TelemetryHandler
|
||||
telemetry = TelemetryHandler(verbose=getattr(self.args, 'verbose', False))
|
||||
telemetry.post_process_result(result, True)
|
||||
|
||||
# Flow Display - End workflow
|
||||
if hasattr(self, 'args') and getattr(self.args, 'flow_display', False):
|
||||
from .features.flow_display import FlowDisplayHandler
|
||||
@@ -3236,6 +3520,207 @@ Now, {final_instruction.lower()}:"""
|
||||
print("pip install praisonaiagents # For PraisonAIAgents\n")
|
||||
sys.exit(1)
|
||||
|
||||
def _run_with_status_display(self, agent, prompt):
|
||||
"""
|
||||
Run agent with minimal status display (spinner + tool/handoff updates).
|
||||
|
||||
Shows:
|
||||
- "Generating..." with spinner while processing
|
||||
- Real-time tool call notifications via registered callback
|
||||
- Agent handoff notifications
|
||||
"""
|
||||
import threading
|
||||
import time
|
||||
from rich.console import Console
|
||||
from rich.live import Live
|
||||
from rich.text import Text
|
||||
|
||||
# Import callback registration from praisonaiagents
|
||||
from praisonaiagents import register_display_callback, sync_display_callbacks
|
||||
|
||||
console = Console()
|
||||
|
||||
# Get tool names for display
|
||||
tool_names = []
|
||||
if hasattr(agent, 'tools') and agent.tools:
|
||||
for tool in agent.tools:
|
||||
if hasattr(tool, '__name__'):
|
||||
tool_names.append(tool.__name__)
|
||||
elif hasattr(tool, 'name'):
|
||||
tool_names.append(tool.name)
|
||||
|
||||
status_info = {
|
||||
'status': 'Generating...',
|
||||
'tool_calls': [],
|
||||
'handoffs': [],
|
||||
'done': False,
|
||||
'result': None,
|
||||
'error': None,
|
||||
'start_time': time.time(),
|
||||
'available_tools': tool_names,
|
||||
'approval_pending': False, # Flag to pause Live display during approval
|
||||
'live_instance': None # Reference to Live instance for stopping
|
||||
}
|
||||
|
||||
def tool_call_callback(message):
|
||||
"""Callback triggered when a tool is called."""
|
||||
# Extract tool name from message
|
||||
if "Calling function:" in message:
|
||||
# Format: "Calling function: function_name"
|
||||
parts = message.split("Calling function:")
|
||||
if len(parts) > 1:
|
||||
tool_name = parts[1].strip()
|
||||
if tool_name and tool_name not in status_info['tool_calls']:
|
||||
status_info['tool_calls'].append(tool_name)
|
||||
status_info['status'] = f"Using {tool_name}..."
|
||||
elif "Function " in message and " returned:" in message:
|
||||
# Format: "Function function_name returned: ..."
|
||||
# Tool execution completed, update status
|
||||
status_info['status'] = "Processing result..."
|
||||
|
||||
# Register callback for tool calls
|
||||
register_display_callback('tool_call', tool_call_callback)
|
||||
|
||||
def build_status_display():
|
||||
"""Build the status display text."""
|
||||
elapsed = time.time() - status_info['start_time']
|
||||
|
||||
# Main status with spinner
|
||||
text = Text()
|
||||
text.append("⏳ ", style="cyan")
|
||||
text.append(f"{status_info['status']} ", style="bold")
|
||||
text.append(f"({elapsed:.1f}s)", style="dim")
|
||||
|
||||
# Show available tools on first line
|
||||
if status_info['available_tools'] and not status_info['tool_calls']:
|
||||
tools_str = ', '.join(status_info['available_tools'][:3])
|
||||
if len(status_info['available_tools']) > 3:
|
||||
tools_str += f" +{len(status_info['available_tools']) - 3} more"
|
||||
text.append(f"\n 🔧 Tools: {tools_str}", style="dim")
|
||||
|
||||
# Show recent tool calls
|
||||
if status_info['tool_calls']:
|
||||
text.append("\n")
|
||||
for tool in status_info['tool_calls'][-3:]: # Show last 3
|
||||
text.append(f" ⚙ {tool}", style="dim yellow")
|
||||
text.append("\n")
|
||||
|
||||
# Show handoffs
|
||||
if status_info['handoffs']:
|
||||
for handoff in status_info['handoffs'][-2:]: # Show last 2
|
||||
text.append(f" → {handoff}", style="dim cyan")
|
||||
text.append("\n")
|
||||
|
||||
return text
|
||||
|
||||
def run_agent():
|
||||
"""Run the agent in background thread."""
|
||||
try:
|
||||
status_info['result'] = agent.start(prompt)
|
||||
except Exception as e:
|
||||
status_info['error'] = e
|
||||
finally:
|
||||
status_info['done'] = True
|
||||
|
||||
# Set up approval callback that stops Live display before prompting
|
||||
# Only if not using --trust (which auto-approves everything)
|
||||
if not getattr(self.args, 'trust', False):
|
||||
from praisonaiagents.approval import set_approval_callback, ApprovalDecision
|
||||
from rich.prompt import Confirm
|
||||
from rich.panel import Panel
|
||||
|
||||
def cli_approval_with_live_pause(function_name, arguments, risk_level):
|
||||
"""Approval callback that stops Live display before prompting."""
|
||||
# Signal to stop Live display
|
||||
status_info['approval_pending'] = True
|
||||
|
||||
# Wait a moment for Live to stop
|
||||
time.sleep(0.2)
|
||||
|
||||
# Now show the approval prompt
|
||||
risk_colors = {"critical": "bold red", "high": "red", "medium": "yellow", "low": "blue"}
|
||||
risk_color = risk_colors.get(risk_level, "white")
|
||||
|
||||
tool_info = f"[bold]Function:[/] {function_name}\n"
|
||||
tool_info += f"[bold]Risk Level:[/] [{risk_color}]{risk_level.upper()}[/{risk_color}]\n"
|
||||
tool_info += "[bold]Arguments:[/]\n"
|
||||
for key, value in arguments.items():
|
||||
str_value = str(value)[:100] + "..." if len(str(value)) > 100 else str(value)
|
||||
tool_info += f" {key}: {str_value}\n"
|
||||
|
||||
console.print(Panel(tool_info.strip(), title="🔒 Tool Approval Required", border_style=risk_color))
|
||||
|
||||
try:
|
||||
approved = Confirm.ask(f"[{risk_color}]Execute this {risk_level} risk tool?[/{risk_color}]", default=False)
|
||||
status_info['approval_pending'] = False
|
||||
|
||||
if approved:
|
||||
console.print("[green]✅ Approved[/green]")
|
||||
return ApprovalDecision(approved=True, reason="User approved")
|
||||
else:
|
||||
console.print("[red]❌ Denied[/red]")
|
||||
return ApprovalDecision(approved=False, reason="User denied")
|
||||
except (KeyboardInterrupt, EOFError):
|
||||
status_info['approval_pending'] = False
|
||||
console.print("\n[red]❌ Cancelled[/red]")
|
||||
return ApprovalDecision(approved=False, reason="User cancelled")
|
||||
|
||||
# Only set if not already set by --approve-level
|
||||
if not getattr(self.args, 'approve_level', None):
|
||||
set_approval_callback(cli_approval_with_live_pause)
|
||||
|
||||
# Start agent in background thread
|
||||
thread = threading.Thread(target=run_agent, daemon=True)
|
||||
thread.start()
|
||||
|
||||
# Show live status while processing
|
||||
try:
|
||||
with Live(build_status_display(), console=console, refresh_per_second=4, transient=True) as live:
|
||||
status_info['live_instance'] = live
|
||||
while not status_info['done']:
|
||||
# Check if approval is pending - stop Live to show prompt
|
||||
if status_info['approval_pending']:
|
||||
break
|
||||
live.update(build_status_display())
|
||||
time.sleep(0.1)
|
||||
|
||||
# If approval was pending, wait for it to complete then restart Live
|
||||
while status_info['approval_pending']:
|
||||
time.sleep(0.1)
|
||||
|
||||
# Continue with Live display if not done
|
||||
if not status_info['done']:
|
||||
with Live(build_status_display(), console=console, refresh_per_second=4, transient=True) as live:
|
||||
while not status_info['done']:
|
||||
if status_info['approval_pending']:
|
||||
break
|
||||
live.update(build_status_display())
|
||||
time.sleep(0.1)
|
||||
except KeyboardInterrupt:
|
||||
console.print("\n[dim]Interrupted[/dim]")
|
||||
# Unregister callback
|
||||
if 'tool_call' in sync_display_callbacks:
|
||||
del sync_display_callbacks['tool_call']
|
||||
return None
|
||||
|
||||
# Wait for thread to complete
|
||||
thread.join(timeout=1.0)
|
||||
|
||||
# Unregister callback to avoid memory leaks
|
||||
if 'tool_call' in sync_display_callbacks:
|
||||
del sync_display_callbacks['tool_call']
|
||||
|
||||
# Handle result
|
||||
if status_info['error']:
|
||||
console.print(f"[red]Error: {status_info['error']}[/red]")
|
||||
return None
|
||||
|
||||
# Show tool calls that were made (if any)
|
||||
if status_info['tool_calls']:
|
||||
console.print(f"[dim]Tools used: {', '.join(status_info['tool_calls'])}[/dim]")
|
||||
|
||||
return status_info['result']
|
||||
|
||||
def _handle_serve_command(self, args, unknown_args):
|
||||
"""
|
||||
Handle the serve command - start API server for agents.
|
||||
@@ -3845,22 +4330,23 @@ Now, {final_instruction.lower()}:"""
|
||||
prompt_session = None
|
||||
try:
|
||||
from prompt_toolkit import PromptSession
|
||||
from prompt_toolkit.completion import WordCompleter
|
||||
from prompt_toolkit.auto_suggest import AutoSuggestFromHistory
|
||||
from prompt_toolkit.history import InMemoryHistory
|
||||
from praisonai.cli.features.at_mentions import CombinedCompleter
|
||||
|
||||
# Create completer for slash commands
|
||||
# Create combined completer for / commands and @ mentions
|
||||
commands = ['help', 'exit', 'quit', 'clear', 'tools', 'profile', 'model', 'stats', 'compact', 'undo', 'queue', 'q']
|
||||
slash_completer = WordCompleter(
|
||||
[f'/{cmd}' for cmd in commands],
|
||||
ignore_case=True
|
||||
combined_completer = CombinedCompleter(
|
||||
commands=commands,
|
||||
root_dir=os.getcwd()
|
||||
)
|
||||
|
||||
prompt_session = PromptSession(
|
||||
message="❯ ",
|
||||
completer=slash_completer,
|
||||
completer=combined_completer,
|
||||
auto_suggest=AutoSuggestFromHistory(),
|
||||
history=InMemoryHistory()
|
||||
history=InMemoryHistory(),
|
||||
complete_while_typing=True
|
||||
)
|
||||
except ImportError:
|
||||
pass # Fall back to simple input
|
||||
@@ -4334,16 +4820,14 @@ Provide a concise summary (max 200 words):"""
|
||||
|
||||
def _process_interactive_prompt_async(self, prompt, tools_list, console, session_state=None):
|
||||
"""
|
||||
Process a prompt asynchronously with live status display.
|
||||
Process a prompt asynchronously with non-blocking status display.
|
||||
|
||||
Shows real-time status (tools, commands) while processing in background.
|
||||
Shows status updates without blocking user input.
|
||||
Allows user to queue more messages while processing.
|
||||
"""
|
||||
import threading
|
||||
import time
|
||||
from rich.live import Live
|
||||
from rich.panel import Panel
|
||||
from rich.text import Text
|
||||
import sys
|
||||
from praisonai.cli.features.message_queue import ProcessingState
|
||||
|
||||
state_manager = session_state['state_manager']
|
||||
@@ -4355,8 +4839,11 @@ Provide a concise summary (max 200 words):"""
|
||||
live_status.clear()
|
||||
live_status.update_status("Thinking...")
|
||||
|
||||
# Print initial status (non-blocking - user can still type)
|
||||
console.print(f"[dim cyan]⏳ {live_status.current_status}[/dim cyan]")
|
||||
|
||||
# Result container for thread communication
|
||||
result_container = {'response': None, 'error': None, 'done': False}
|
||||
result_container = {'response': None, 'error': None, 'done': False, 'last_status': ''}
|
||||
|
||||
def process_in_background():
|
||||
"""Run agent processing in background thread."""
|
||||
@@ -4402,37 +4889,36 @@ Provide a concise summary (max 200 words):"""
|
||||
bg_thread = threading.Thread(target=process_in_background, daemon=True)
|
||||
bg_thread.start()
|
||||
|
||||
# Show live status while processing
|
||||
try:
|
||||
with Live(console=console, refresh_per_second=4, transient=True) as live:
|
||||
while not result_container['done']:
|
||||
# Build status display
|
||||
status_text = Text()
|
||||
status_text.append("⏳ ", style="cyan")
|
||||
status_text.append(live_status.current_status or "Processing...", style="dim")
|
||||
|
||||
# Show queued messages if any
|
||||
queue_count = message_queue.count
|
||||
if queue_count > 0:
|
||||
status_text.append(f"\n📋 Queued: {queue_count}", style="dim yellow")
|
||||
|
||||
live.update(Panel(status_text, border_style="dim"))
|
||||
time.sleep(0.1)
|
||||
except KeyboardInterrupt:
|
||||
console.print("\n[dim]Processing interrupted[/dim]")
|
||||
# Wait for completion with periodic status updates (non-blocking style)
|
||||
start_time = time.time()
|
||||
last_status_update = ""
|
||||
while not result_container['done']:
|
||||
time.sleep(0.2)
|
||||
current_status = live_status.current_status
|
||||
# Only print status if it changed
|
||||
if current_status != last_status_update:
|
||||
# Use carriage return to update in place
|
||||
elapsed = time.time() - start_time
|
||||
queue_count = message_queue.count
|
||||
queue_info = f" | 📋 Queued: {queue_count}" if queue_count > 0 else ""
|
||||
sys.stdout.write(f"\r[dim cyan]⏳ {current_status} ({elapsed:.1f}s){queue_info}[/dim cyan]")
|
||||
sys.stdout.flush()
|
||||
last_status_update = current_status
|
||||
|
||||
# Clear the status line
|
||||
sys.stdout.write("\r" + " " * 80 + "\r")
|
||||
sys.stdout.flush()
|
||||
|
||||
# Wait for thread to finish
|
||||
bg_thread.join(timeout=1.0)
|
||||
|
||||
# Handle result
|
||||
if result_container['error']:
|
||||
console.print(f"\n[red]Error: {result_container['error']}[/red]")
|
||||
console.print(f"[red]Error: {result_container['error']}[/red]")
|
||||
elif result_container['response']:
|
||||
# Print response with streaming effect
|
||||
console.print()
|
||||
response_str = result_container['response']
|
||||
words = response_str.split()
|
||||
import sys
|
||||
for i, word in enumerate(words):
|
||||
console.print(word + " ", end="")
|
||||
sys.stdout.flush()
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
"""
|
||||
Simple CLI entry point for praisonai schedule command.
|
||||
|
||||
This provides a standalone entry point that can be called from main.py
|
||||
"""
|
||||
|
||||
import sys
|
||||
from praisonai.cli.features.agent_scheduler import AgentSchedulerHandler
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point for schedule CLI command."""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Schedule an agent to run continuously at regular intervals'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'schedule_yaml',
|
||||
nargs='?',
|
||||
default='agents.yaml',
|
||||
help='Path to agents.yaml file (default: agents.yaml)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--interval',
|
||||
dest='schedule_interval',
|
||||
type=str,
|
||||
help='Override schedule interval (e.g., "hourly", "*/30m", "daily")'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--max-retries',
|
||||
dest='schedule_max_retries',
|
||||
type=int,
|
||||
help='Override maximum retry attempts (default: from YAML or 3)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--verbose', '-v',
|
||||
action='store_true',
|
||||
help='Enable verbose logging'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Call the handler
|
||||
exit_code = AgentSchedulerHandler.handle_schedule_command(args)
|
||||
sys.exit(exit_code)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -57,7 +57,7 @@ class CloudDeployer:
|
||||
file.write("FROM python:3.11-slim\n")
|
||||
file.write("WORKDIR /app\n")
|
||||
file.write("COPY . .\n")
|
||||
file.write("RUN pip install flask praisonai==2.3.52 gunicorn markdown\n")
|
||||
file.write("RUN pip install flask praisonai==2.3.85 gunicorn markdown\n")
|
||||
file.write("EXPOSE 8080\n")
|
||||
file.write('CMD ["gunicorn", "-b", "0.0.0.0:8080", "api:app"]\n')
|
||||
|
||||
|
||||
@@ -135,7 +135,8 @@ class BaseCLIIntegration(ABC):
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
cwd=self.workspace
|
||||
cwd=self.workspace,
|
||||
env=self.get_env()
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -173,7 +174,8 @@ class BaseCLIIntegration(ABC):
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
cwd=self.workspace
|
||||
cwd=self.workspace,
|
||||
env=self.get_env()
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -208,7 +210,8 @@ class BaseCLIIntegration(ABC):
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
cwd=self.workspace
|
||||
cwd=self.workspace,
|
||||
env=self.get_env()
|
||||
)
|
||||
|
||||
try:
|
||||
|
||||
@@ -127,23 +127,16 @@ class ClaudeCodeIntegration(BaseCLIIntegration):
|
||||
"""
|
||||
cmd = ["claude"]
|
||||
|
||||
# Add continue flag if needed
|
||||
if continue_session or self._session_active:
|
||||
cmd.append("--continue")
|
||||
|
||||
# Add skip permissions flag
|
||||
if self.skip_permissions:
|
||||
cmd.append("--dangerously-skip-permissions")
|
||||
|
||||
# Add print mode flag
|
||||
# Add print mode flag for non-interactive output
|
||||
cmd.append("-p")
|
||||
|
||||
# Add prompt
|
||||
cmd.append(prompt)
|
||||
|
||||
# Add output format
|
||||
cmd.extend(["--output-format", self.output_format])
|
||||
|
||||
# Add continue flag if needed
|
||||
if continue_session or self._session_active:
|
||||
cmd.append("--continue")
|
||||
|
||||
# Add model if specified
|
||||
if self.model:
|
||||
cmd.extend(["--model", self.model])
|
||||
@@ -160,6 +153,13 @@ class ClaudeCodeIntegration(BaseCLIIntegration):
|
||||
if self.disallowed_tools:
|
||||
cmd.extend(["--disallowedTools", ",".join(self.disallowed_tools)])
|
||||
|
||||
# Add verbose if needed
|
||||
if options.get('verbose'):
|
||||
cmd.append("--verbose")
|
||||
|
||||
# Add prompt last
|
||||
cmd.append(prompt)
|
||||
|
||||
return cmd
|
||||
|
||||
async def execute(self, prompt: str, **options) -> str:
|
||||
|
||||
@@ -93,6 +93,9 @@ class CodexCLIIntegration(BaseCLIIntegration):
|
||||
"""
|
||||
cmd = ["codex", "exec"]
|
||||
|
||||
# Add working directory
|
||||
cmd.extend(["-C", self.workspace])
|
||||
|
||||
# Add task
|
||||
cmd.append(task)
|
||||
|
||||
|
||||
@@ -97,6 +97,9 @@ class CursorCLIIntegration(BaseCLIIntegration):
|
||||
# Add print mode flag
|
||||
cmd.append("-p")
|
||||
|
||||
# Add workspace
|
||||
cmd.extend(["--workspace", self.workspace])
|
||||
|
||||
# Add force flag if enabled
|
||||
if self.force:
|
||||
cmd.append("--force")
|
||||
|
||||
@@ -92,12 +92,6 @@ class GeminiCLIIntegration(BaseCLIIntegration):
|
||||
"""
|
||||
cmd = ["gemini"]
|
||||
|
||||
# Add print mode flag
|
||||
cmd.append("-p")
|
||||
|
||||
# Add prompt
|
||||
cmd.append(prompt)
|
||||
|
||||
# Add model
|
||||
cmd.extend(["-m", self.model])
|
||||
|
||||
@@ -112,6 +106,12 @@ class GeminiCLIIntegration(BaseCLIIntegration):
|
||||
if self.sandbox:
|
||||
cmd.append("--sandbox")
|
||||
|
||||
# Add YOLO mode for non-interactive execution
|
||||
cmd.append("--yolo")
|
||||
|
||||
# Add prompt flag with prompt value
|
||||
cmd.extend(["-p", prompt])
|
||||
|
||||
return cmd
|
||||
|
||||
async def execute(self, prompt: str, **options) -> str:
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,13 +1,31 @@
|
||||
import re
|
||||
"""
|
||||
DEPRECATED: This module is deprecated. Use praisonai.scheduler instead.
|
||||
|
||||
This file is kept for backward compatibility only.
|
||||
All new code should import from praisonai.scheduler.
|
||||
"""
|
||||
|
||||
import warnings
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Union, Optional, Callable, Dict, Any
|
||||
from typing import Optional, Dict, Any
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Issue deprecation warning
|
||||
warnings.warn(
|
||||
"praisonai.scheduler module (root level) is deprecated. "
|
||||
"Use 'from praisonai.scheduler import ...' instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
|
||||
# Import from new scheduler module for backward compatibility
|
||||
from .scheduler.base import ScheduleParser
|
||||
|
||||
# Keep old classes for backward compatibility
|
||||
class DeployerInterface(ABC):
|
||||
"""Abstract interface for deployers to ensure provider compatibility."""
|
||||
|
||||
@@ -32,45 +50,6 @@ class CloudDeployerAdapter(DeployerInterface):
|
||||
logger.error(f"Deployment failed: {e}")
|
||||
return False
|
||||
|
||||
class ScheduleParser:
|
||||
"""Parse schedule expressions into intervals."""
|
||||
|
||||
@staticmethod
|
||||
def parse(schedule_expr: str) -> int:
|
||||
"""
|
||||
Parse schedule expression and return interval in seconds.
|
||||
|
||||
Supported formats:
|
||||
- "daily" -> 86400 seconds
|
||||
- "hourly" -> 3600 seconds
|
||||
- "*/30m" -> 1800 seconds (every 30 minutes)
|
||||
- "*/6h" -> 21600 seconds (every 6 hours)
|
||||
- "60" -> 60 seconds (plain number)
|
||||
"""
|
||||
schedule_expr = schedule_expr.strip().lower()
|
||||
|
||||
if schedule_expr == "daily":
|
||||
return 86400
|
||||
elif schedule_expr == "hourly":
|
||||
return 3600
|
||||
elif schedule_expr.isdigit():
|
||||
return int(schedule_expr)
|
||||
elif schedule_expr.startswith("*/"):
|
||||
# Handle */30m, */6h patterns
|
||||
interval_part = schedule_expr[2:]
|
||||
if interval_part.endswith("m"):
|
||||
minutes = int(interval_part[:-1])
|
||||
return minutes * 60
|
||||
elif interval_part.endswith("h"):
|
||||
hours = int(interval_part[:-1])
|
||||
return hours * 3600
|
||||
elif interval_part.endswith("s"):
|
||||
return int(interval_part[:-1])
|
||||
else:
|
||||
return int(interval_part)
|
||||
else:
|
||||
raise ValueError(f"Unsupported schedule format: {schedule_expr}")
|
||||
|
||||
class DeploymentScheduler:
|
||||
"""
|
||||
Minimal deployment scheduler with provider-agnostic design.
|
||||
|
||||
Loaded 100 of 133 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user