mirror of
https://github.com/MervinPraison/PraisonAI.git
synced 2026-11-08 17:31:00 -10:00
Compare commits
381
Commits
No files matched your search
@@ -0,0 +1,330 @@
|
||||
<!-- wp:paragraph -->
|
||||
<p>praisonai-mcp is Tier 2e — the heavy MCP server host: full <code>MCPServer</code>, tool/resource/prompt registries, recipe bridge, HTTP-stream auth, and MCP client config management. Depends only on <code>praisonaiagents</code>. v0.0.2. Protocol: MCP 2025-11-25.</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:merpress/mermaidjs {"align":"full"} -->
|
||||
<div class="wp-block-merpress-mermaidjs diagram-source-mermaid alignfull"><pre class="mermaid">%%{init: {"theme": "base", "themeVariables": {"background": "transparent", "lineColor": "#000000"}}}%%
|
||||
flowchart TB
|
||||
CLIENT[Cursor Claude Desktop]
|
||||
subgraph T1 [praisonaiagents]
|
||||
SDK[MCP client ToolsMCPServer]
|
||||
SEC[mcp_security shared]
|
||||
end
|
||||
subgraph T2e [praisonai-mcp]
|
||||
HOST[MCPServer]
|
||||
REG[tool resource prompt registry]
|
||||
TR[stdio http-stream]
|
||||
AUTH[api-key oauth scopes]
|
||||
CFG[mcp list add sync]
|
||||
end
|
||||
subgraph T2a [praisonai-code optional]
|
||||
LIGHT[serve mcp light]
|
||||
BRIDGE[_mcp_bridge]
|
||||
end
|
||||
subgraph T3 [praisonai wrapper optional]
|
||||
CAP[capabilities recipe]
|
||||
end
|
||||
CLIENT --> TR
|
||||
TR --> HOST
|
||||
HOST --> REG
|
||||
REG -.->|adapters| CAP
|
||||
CFG -.->|_code_bridge| LIGHT
|
||||
BRIDGE --> CFG
|
||||
SDK --> LIGHT
|
||||
SEC --> TR
|
||||
classDef agent fill:#8B0000,color:#fff
|
||||
classDef tool fill:#189AB4,color:#fff
|
||||
class CLIENT agent
|
||||
class SDK agent
|
||||
class HOST agent
|
||||
class REG tool
|
||||
class TR tool
|
||||
class AUTH tool
|
||||
class CFG tool
|
||||
class LIGHT tool
|
||||
class SEC tool
|
||||
</pre></div>
|
||||
<!-- /wp:merpress/mermaidjs -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Three MCP layers (do not conflate)</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Layer</th><th>Package</th><th>Role</th><th>Entry</th></tr></thead><tbody><tr><td>Client</td><td><code>praisonaiagents/mcp/</code></td><td>Connect agents to external MCP servers; OAuth storage</td><td><code>pip install "praisonaiagents[mcp]"</code></td></tr><tr><td>Light server</td><td><code>praisonai-code</code></td><td>Config-driven basic tool hosting</td><td><code>praisonai serve mcp</code></td></tr><tr><td>Heavy host</td><td><code>praisonai-mcp</code></td><td>Full server, adapters, recipe bridge, HTTP auth</td><td><code>praisonai mcp serve</code></td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Install matrix</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Install</th><th><code>praisonai mcp</code></th><th><code>praisonai serve mcp</code></th><th><code>praisonai-mcp</code> script</th></tr></thead><tbody><tr><td><code>pip install praisonai-mcp</code></td><td>—</td><td>—</td><td>✅</td></tr><tr><td><code>pip install "praisonai-mcp[all]"</code></td><td>—</td><td>—</td><td>✅ HTTP + auth</td></tr><tr><td><code>pip install praisonai-code</code> only</td><td>hidden</td><td>✅ light</td><td>—</td></tr><tr><td><code>pip install "praisonai[mcp]"</code></td><td>✅</td><td>✅</td><td>✅</td></tr><tr><td><code>pip install praisonai</code></td><td>✅</td><td>✅</td><td>✅</td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>1. Standalone heavy host</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>pip install "praisonai-mcp[all]"
|
||||
export OPENAI_API_KEY=sk-...
|
||||
|
||||
praisonai-mcp --version
|
||||
praisonai-mcp serve --transport stdio
|
||||
praisonai-mcp list-tools
|
||||
praisonai-mcp doctor</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>2. Full stack (same commands via <code>praisonai mcp …</code>)</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>pip install praisonai
|
||||
# or MCP-only extra
|
||||
pip install "praisonai[mcp]"
|
||||
|
||||
praisonai mcp serve --transport stdio
|
||||
praisonai mcp list-tools
|
||||
praisonai mcp doctor</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>3. Light server only (no praisonai-mcp required)</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>pip install praisonai-code "praisonaiagents[mcp]"
|
||||
|
||||
praisonai serve mcp --port 8080
|
||||
# Uses agents ToolsMCPServer — not the heavy MCPServer</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:separator -->
|
||||
<hr class="wp-block-separator has-alpha-channel-opacity"/>
|
||||
<!-- /wp:separator -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 1 — STDIO host (Cursor / Claude Desktop)</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code># Heavy host — full PraisonAI tool registry
|
||||
praisonai-mcp serve --transport stdio
|
||||
# or
|
||||
praisonai mcp serve --transport stdio
|
||||
|
||||
# Generate client config
|
||||
praisonai mcp config-generate --client cursor --transport stdio
|
||||
praisonai mcp config-generate --client claude-desktop --transport stdio</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>STDIO uses JSON-RPC over stdin/stdout. Logs go to stderr only — never corrupt the protocol stream.</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 2 — HTTP Stream host</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Streamable HTTP (MCP 2025-11-25) — not legacy standalone SSE. Requires <code>[server]</code> extra (starlette + uvicorn).</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>praisonai mcp serve \
|
||||
--transport http-stream \
|
||||
--host 127.0.0.1 \
|
||||
--port 8080 \
|
||||
--endpoint /mcp \
|
||||
--api-key my-secret-key \
|
||||
--response-mode batch
|
||||
|
||||
# Endpoints: POST/GET/DELETE /mcp, plus /health</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Non-localhost binds require <code>--api-key</code> or <code>--keys-file</code>. Origin validation uses shared helpers from <code>praisonaiagents.mcp.mcp_security</code>.</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 3 — Recipe → MCP bridge</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Turn any PraisonAI recipe into a scoped MCP server with allow/deny lists and safe mode.</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>praisonai mcp serve-recipe support-reply --transport stdio
|
||||
|
||||
praisonai mcp validate-recipe support-reply
|
||||
praisonai mcp inspect-recipe support-reply
|
||||
|
||||
praisonai mcp config-generate-recipe support-reply --client claude-desktop</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Default denied tools include <code>shell.exec</code>, <code>file.delete</code>, <code>eval</code>, and similar destructive operations.</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 4 — MCP client config management</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Config subcommands (<code>list</code>, <code>add</code>, <code>remove</code>, <code>sync</code>, <code>test</code>) require <code>praisonai-code</code> via <code>_code_bridge</code>.</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>praisonai mcp list
|
||||
praisonai mcp add filesystem \
|
||||
--command npx \
|
||||
--args "-y @modelcontextprotocol/server-filesystem ."
|
||||
|
||||
praisonai mcp test filesystem
|
||||
praisonai mcp sync
|
||||
praisonai mcp tools
|
||||
praisonai mcp status</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 5 — Built-in registry (tools, resources, prompts)</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Type</th><th>Examples</th><th>Adapter</th></tr></thead><tbody><tr><td>Tools</td><td><code>praisonai.chat.*</code>, <code>praisonai.workflow.*</code>, agent run helpers</td><td><code>mcp_server/adapters/capabilities.py</code>, <code>cli_tools.py</code></td></tr><tr><td>Resources</td><td><code>praisonai://memory/sessions</code>, <code>praisonai://workflows</code>, <code>praisonai://agents</code></td><td><code>adapters/resources.py</code></td></tr><tr><td>Prompts</td><td><code>deep-research</code>, <code>code-review</code>, <code>workflow-auto</code></td><td><code>adapters/prompts.py</code></td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code># Inspect what the host exposes
|
||||
praisonai-mcp list-tools
|
||||
praisonai-mcp list-resources
|
||||
praisonai-mcp list-prompts
|
||||
|
||||
# Host tool search (MCPServerCLI)
|
||||
praisonai-mcp tools search "workflow"</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 6 — Security & auth</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Concern</th><th>Package</th><th>Module</th></tr></thead><tbody><tr><td>Origin validation, session IDs</td><td><code>praisonaiagents</code></td><td><code>mcp/mcp_security.py</code></td></tr><tr><td>API keys, scoped keys file</td><td><code>praisonai-mcp</code></td><td><code>mcp_server/auth/api_key.py</code></td></tr><tr><td>OAuth 2.1 + PKCE</td><td><code>praisonai-mcp</code></td><td><code>mcp_server/auth/oauth.py</code></td></tr><tr><td>OIDC discovery</td><td><code>praisonai-mcp</code></td><td><code>mcp_server/auth/oidc.py</code></td></tr><tr><td>Scope enforcement</td><td><code>praisonai-mcp</code></td><td><code>mcp_server/auth/scopes.py</code></td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code># Scoped keys file (--keys-file)
|
||||
[
|
||||
{"key": "monitor-key", "name": "monitor", "scopes": ["tools:read", "resources:read"]},
|
||||
{"key": "admin-key", "name": "admin", "scopes": ["*"]}
|
||||
]
|
||||
|
||||
# Env vars
|
||||
export MCP_API_KEY=your-key
|
||||
export PRAISONAI_MCP_MAX_SESSIONS=1000</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 7 — CLI routing (lazy load)</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Registry</th><th>Commands</th><th>Gate</th></tr></thead><tbody><tr><td>MCP-resident Typer</td><td>list, add, remove, sync, test, serve, doctor</td><td><code>mcp_package_available()</code></td></tr><tr><td>MCPServerCLI host</td><td>serve, list-tools, list-resources, config-generate, tasks, auth</td><td>Direct via <code>praisonai-mcp <cmd></code></td></tr><tr><td>Code-resident light</td><td><code>serve mcp</code></td><td>Always available with code + agents</td></tr><tr><td>Wrapper-resident</td><td>dashboard, schedule, langfuse</td><td><code>wrapper_package_available()</code></td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Standalone <code>praisonai-code</code> hides <code>mcp</code> from <code>--help</code> when <code>praisonai-mcp</code> is not installed. Host commands like <code>praisonai-mcp serve</code> skip the <code>mcp</code> prefix; <code>praisonai mcp serve</code> goes through the Typer group.</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 8 — Python API</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>import praisonai_mcp
|
||||
from praisonai_mcp.mcp_server import MCPServer
|
||||
from praisonai_mcp.mcp_server.registry import register_tool
|
||||
from praisonai_mcp.mcp_server.adapters import register_all
|
||||
|
||||
@register_tool("custom.greet")
|
||||
def greet(name: str) -> str:
|
||||
"""Greet a person by name."""
|
||||
return f"Hello, {name}!"
|
||||
|
||||
register_all()
|
||||
server = MCPServer(name="praisonai")
|
||||
server.run(transport="stdio")</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code># Recipe host
|
||||
from praisonai_mcp.mcp_server import RecipeMCPAdapter
|
||||
|
||||
adapter = RecipeMCPAdapter("support-reply")
|
||||
adapter.load()
|
||||
adapter.to_mcp_server().run(transport="stdio")
|
||||
|
||||
# Shim path still works
|
||||
from praisonai.mcp_server import MCPServer</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 9 — Standalone (no wrapper)</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>pip install praisonaiagents "praisonai-mcp[all]"
|
||||
export OPENAI_API_KEY=sk-...
|
||||
|
||||
praisonai-mcp serve --transport stdio
|
||||
praisonai-mcp list-tools
|
||||
praisonai-mcp doctor
|
||||
praisonai-mcp --help</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Heavy MCP hosting needs only <code>praisonaiagents</code> + <code>praisonai-mcp</code>. Full capability registry (workflows, deploy, extended tools) activates when the wrapper is co-installed via <code>_wrapper_bridge</code>.</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Bridges (no PyPI cycles)</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Bridge</th><th>Direction</th><th>Use</th></tr></thead><tbody><tr><td><code>_mcp_bridge</code></td><td>code → mcp</td><td>Route <code>praisonai mcp</code> into Typer group</td></tr><tr><td><code>_code_bridge</code></td><td>mcp → code</td><td>Config loader, paths, output controller</td></tr><tr><td><code>_wrapper_bridge</code></td><td>mcp → wrapper</td><td>Capabilities, recipe, deploy adapters</td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Backward-compat shims</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Old path</th><th>Still works via</th></tr></thead><tbody><tr><td><code>praisonai.mcp_server.*</code></td><td><code>alias_package</code> → <code>praisonai_mcp.mcp_server</code></td></tr><tr><td><code>praisonai.cli.commands.mcp</code></td><td><code>sys.modules</code> shim → mcp Typer group</td></tr><tr><td><code>from praisonai.mcp_server import MCPServer</code></td><td>Lazy re-export through shim</td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Seven-package publish order</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>praisonaiagents → praisonai-code → praisonai-bot → praisonai-train → praisonai-browser → praisonai-mcp → praisonai (wrapper)</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Wrapper pins <code>praisonai-mcp>=X</code> after mcp is on PyPI. CI: <code>.github/workflows/pypi-release.yml</code>. Import gate: <code>scripts/check_c12_mcp_imports.sh</code>.</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Summary</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Item</th><th>Value</th></tr></thead><tbody><tr><td>Tier</td><td>2e</td></tr><tr><td>PyPI</td><td><code>praisonai-mcp</code> v0.0.2</td></tr><tr><td>Runtime deps</td><td><code>praisonaiagents</code>, <code>mcp</code>, rich, typer</td></tr><tr><td>HTTP extra</td><td><code>[server]</code> — starlette, uvicorn</td></tr><tr><td>Auth extra</td><td><code>[auth]</code> — httpx</td></tr><tr><td>Transports</td><td><code>stdio</code>, <code>http-stream</code></td></tr><tr><td>Light alternative</td><td><code>praisonai serve mcp</code> (code tier)</td></tr><tr><td>CLI entry</td><td><code>praisonai-mcp</code> or <code>praisonai mcp</code></td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Previous: <a href="https://mer.vin/2026/07/praisonai-train-praisonai-train-fine-tuning-agent-training-in-depth/">praisonai-train</a> · Next: praisonai wrapper</p>
|
||||
<!-- /wp:paragraph -->
|
||||
@@ -0,0 +1,460 @@
|
||||
<!-- wp:paragraph -->
|
||||
<p>praisonai is Tier 3 — the integration wrapper that wires <code>praisonaiagents</code> + five Tier-2 packages (<code>praisonai-code</code>, <code>praisonai-bot</code>, <code>praisonai-train</code>, <code>praisonai-browser</code>, <code>praisonai-mcp</code>) into one install, with dashboard, framework adapters, jobs API, serve orchestration, and backward-compat shims. v4.6.150.</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:merpress/mermaidjs {"align":"full"} -->
|
||||
<div class="wp-block-merpress-mermaidjs diagram-source-mermaid alignfull"><pre class="mermaid">%%{init: {"theme": "base", "themeVariables": {"background": "transparent", "lineColor": "#000000"}}}%%
|
||||
flowchart TB
|
||||
subgraph T1 [Tier 1 praisonaiagents]
|
||||
A[Agent tools memory MCP client]
|
||||
end
|
||||
subgraph T2 [Tier 2 packages]
|
||||
C[praisonai-code run chat code]
|
||||
B[praisonai-bot gateway channels]
|
||||
TR[praisonai-train fine-tune agents]
|
||||
BR[praisonai-browser CDP Playwright]
|
||||
MCP[praisonai-mcp server host]
|
||||
end
|
||||
subgraph T3 [Tier 3 praisonai wrapper]
|
||||
W[dashboard jobs frameworks serve shims]
|
||||
end
|
||||
A --> C
|
||||
A --> B
|
||||
A --> TR
|
||||
A --> BR
|
||||
A --> MCP
|
||||
W --> C
|
||||
W --> B
|
||||
W --> TR
|
||||
W --> BR
|
||||
W --> MCP
|
||||
W --> A
|
||||
classDef agent fill:#8B0000,color:#fff
|
||||
classDef tool fill:#189AB4,color:#fff
|
||||
class A agent
|
||||
class W agent
|
||||
class C tool
|
||||
class B tool
|
||||
class TR tool
|
||||
class BR tool
|
||||
class MCP tool
|
||||
</pre></div>
|
||||
<!-- /wp:merpress/mermaidjs -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Seven-package stack</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>praisonaiagents → praisonai-code + praisonai-bot + praisonai-train + praisonai-browser + praisonai-mcp → praisonai (wrapper)</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Each Tier-2 package is installable standalone with its own console script. The wrapper pins all of them and preserves every old import path via shims. Use the wrapper when you need several surfaces together — HTTP serve, dashboard, framework YAML, jobs/recipe — without managing tier pins yourself.</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Product catalog (C0–C12 shipped)</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Package</th><th>Epic</th><th>Choose this when…</th><th>Standalone install</th></tr></thead><tbody><tr><td><code>praisonaiagents</code></td><td>Tier 1</td><td>Building or embedding agents in code — Agent, tools, memory, hooks</td><td><code>pip install praisonaiagents</code></td></tr><tr><td><code>praisonai-code</code></td><td>C0–C8</td><td>Terminal agentic CLI — run, chat, code, warm daemon</td><td><code>pip install praisonai-code</code></td></tr><tr><td><code>praisonai-bot</code></td><td>C9</td><td>Messaging bots + WebSocket gateway</td><td><code>pip install "praisonai-bot[gateway,bot]"</code></td></tr><tr><td><code>praisonai-train</code></td><td>C10</td><td>Agent training loops + GPU LLM fine-tuning</td><td><code>pip install praisonai-train</code> or <code>[llm]</code></td></tr><tr><td><code>praisonai-browser</code></td><td>C11</td><td>Chrome extension bridge, CDP, Playwright agents</td><td><code>pip install "praisonai-browser[all]"</code></td></tr><tr><td><code>praisonai-mcp</code></td><td>C12</td><td>MCP server for Cursor, Claude Desktop, Windsurf</td><td><code>pip install "praisonai-mcp[all]"</code></td></tr><tr><td><code>praisonai</code></td><td>Tier 3</td><td>Full product / umbrella — serve, dashboard, frameworks, jobs</td><td><code>pip install praisonai</code></td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Three-layer products (do not conflate)</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Product</th><th>Layer 1 (agents)</th><th>Layer 2 (dedicated pkg)</th><th>Layer 3 (wrapper)</th></tr></thead><tbody><tr><td>Browser</td><td><code>tools/protocols/browser.py</code></td><td><code>praisonai-browser</code></td><td>shims + optional full stack</td></tr><tr><td>MCP</td><td><code>praisonaiagents/mcp</code> client + light server</td><td><code>praisonai-mcp</code> heavy host</td><td>capability bodies via <code>_wrapper_bridge</code></td></tr><tr><td>Bots</td><td>bots/gateway protocols</td><td><code>praisonai-bot</code></td><td>shims only</td></tr><tr><td>Train</td><td>Agent + hooks in agents</td><td><code>praisonai-train</code></td><td>shims + <code>setup-conda-env</code></td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Install matrix</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>1. Base install (full seven-package stack)</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>pip install praisonai
|
||||
export OPENAI_API_KEY=sk-...
|
||||
|
||||
praisonai doctor
|
||||
praisonai version show</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>2. Targeted extras</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>pip install "praisonai[claw]" # dashboard + bots + gateway
|
||||
pip install "praisonai[all]" # onboarding + messaging + search stack
|
||||
pip install "praisonai[train]" # LLM fine-tuning (Unsloth stack)
|
||||
pip install "praisonai[mcp]" # full MCP server host
|
||||
pip install "praisonai[browser]" # extension bridge + CDP + Playwright
|
||||
pip install "praisonai[crewai]" # multi-framework YAML</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Package</th><th>Min version (wrapper pin)</th><th>Console script</th></tr></thead><tbody><tr><td><code>praisonaiagents</code></td><td>≥ 1.6.153</td><td>—</td></tr><tr><td><code>praisonai-code</code></td><td>≥ 0.0.49</td><td><code>praisonai-code</code></td></tr><tr><td><code>praisonai-bot</code></td><td>≥ 0.0.34</td><td><code>praisonai-bot</code></td></tr><tr><td><code>praisonai-train</code></td><td>≥ 0.0.5</td><td><code>praisonai-train</code></td></tr><tr><td><code>praisonai-browser</code></td><td>≥ 0.0.2</td><td><code>praisonai-browser</code></td></tr><tr><td><code>praisonai-mcp</code></td><td>≥ 0.0.2</td><td><code>praisonai-mcp</code></td></tr><tr><td><code>praisonai</code></td><td>4.6.150</td><td><code>praisonai</code></td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:separator -->
|
||||
<hr class="wp-block-separator has-alpha-channel-opacity"/>
|
||||
<!-- /wp:separator -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 1 — What wrapper adds (not in Tier-2 alone)</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Feature</th><th>Command / path</th><th>Owner</th><th>Extracted?</th></tr></thead><tbody><tr><td>Unified CLI router</td><td><code>praisonai …</code></td><td>code Typer + 6 resident registries</td><td>—</td></tr><tr><td>HTTP serve orchestration</td><td><code>praisonai serve agents|recipe|a2a|…</code></td><td>wrapper <code>cli/features/serve.py</code></td><td>Deferred (C15 epic)</td></tr><tr><td>Dashboard launcher</td><td><code>praisonai dashboard</code></td><td>wrapper</td><td>Stay wrapper</td></tr><tr><td>Framework adapters</td><td><code>praisonai --framework crewai</code></td><td>wrapper <code>framework_adapters/</code></td><td>Stay wrapper (impls in external PraisonAI-Frameworks)</td></tr><tr><td>Jobs API</td><td><code>praisonai.jobs</code> · port 8005</td><td>wrapper</td><td>Deferred (with serve/recipe)</td></tr><tr><td>Recipe NL → YAML</td><td><code>praisonai recipe …</code></td><td>wrapper</td><td>Deferred</td></tr><tr><td>Scheduler CLI</td><td><code>praisonai schedule …</code></td><td>wrapper</td><td>Stay wrapper (3-layer split with bot)</td></tr><tr><td>Sandbox execution</td><td><code>praisonai sandbox …</code></td><td>wrapper <code>sandbox/</code></td><td>C13 candidate</td></tr><tr><td>Cloud deploy</td><td><code>praisonai deploy …</code></td><td>wrapper <code>deploy/</code></td><td>C13 candidate</td></tr><tr><td>Langfuse / Langflow / n8n</td><td><code>langfuse</code>, <code>flow</code>, <code>n8n</code></td><td>wrapper</td><td>Stay wrapper</td></tr><tr><td>Backward-compat shims</td><td><code>praisonai.bots</code> → <code>praisonai_bot</code></td><td>wrapper <code>cli/_shim.py</code></td><td>Permanent</td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Tier-2 packages own their domains; the wrapper owns integration surfaces that span multiple tiers.</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 2 — Terminal hot path (via code tier)</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>pip install praisonai
|
||||
export OPENAI_API_KEY=sk-...
|
||||
|
||||
praisonai run "Summarise the latest AI agent trends"
|
||||
praisonai chat
|
||||
praisonai code
|
||||
praisonai daemon start --background
|
||||
praisonai run "Hello" # auto-forwards to warm runtime</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code># Python SDK (Tier 1 — same everywhere)
|
||||
from praisonaiagents import Agent
|
||||
agent = Agent(instructions="You are a helpful AI assistant")
|
||||
agent.start("Write a movie script about a robot on Mars")</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Deep dive: <a href="https://mer.vin/2026/07/praisonai-code-praisonai-code-terminal-cli-in-depth/">praisonai-code in-depth</a></p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 3 — Train (via praisonai-train tier)</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code># LLM fine-tuning (Unsloth) — lazy ML deps
|
||||
praisonai train llm dataset.json
|
||||
praisonai train llm --model llama-3.1 dataset.json
|
||||
|
||||
# Agent training — LLM-as-judge
|
||||
praisonai train agents --input "What is Python?" --iterations 3
|
||||
|
||||
# Human feedback loop
|
||||
praisonai train agents --input "Explain quantum computing" --human
|
||||
|
||||
# Session management
|
||||
praisonai train list
|
||||
praisonai train apply train-abc123 --run "And Germany?"
|
||||
|
||||
# Standalone script (no wrapper prefix)
|
||||
praisonai-train agents --input "Hello" --iterations 2</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Deep dive: <a href="https://mer.vin/2026/07/praisonai-train-praisonai-train-fine-tuning-agent-training-in-depth/">praisonai-train in-depth</a></p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 4 — Bots & gateway (via bot tier)</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>pip install "praisonai[bot]"
|
||||
export OPENAI_API_KEY=sk-...
|
||||
export TELEGRAM_BOT_TOKEN=...
|
||||
|
||||
praisonai onboard
|
||||
praisonai bot start --platform telegram
|
||||
praisonai gateway start --config ~/.praisonai/bot.yaml
|
||||
praisonai gateway install --config ~/.praisonai/bot.yaml
|
||||
praisonai pairing approve telegram ABC12345</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Deep dive: <a href="https://mer.vin/2026/07/praisonai-bot-praisonai-bot-gateway-channels-in-depth/">praisonai-bot in-depth</a></p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 5 — MCP server host (via mcp tier)</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Command</th><th>Layer</th><th>Package</th></tr></thead><tbody><tr><td><code>praisonai serve mcp</code></td><td>Light server</td><td><code>praisonai-code</code> + agents</td></tr><tr><td><code>praisonai mcp serve</code></td><td>Heavy host</td><td><code>praisonai-mcp</code></td></tr><tr><td><code>praisonai-mcp serve</code></td><td>Heavy host (direct)</td><td><code>praisonai-mcp</code></td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code># Heavy MCP host — full tool/resource/prompt registry
|
||||
praisonai mcp serve --transport stdio
|
||||
praisonai mcp serve --transport http-stream --port 8080 --api-key secret
|
||||
|
||||
# Recipe → scoped MCP server
|
||||
praisonai mcp serve-recipe support-reply --transport stdio
|
||||
|
||||
# Client config management (needs praisonai-code co-installed)
|
||||
praisonai mcp list
|
||||
praisonai mcp add filesystem --command npx --args "-y @modelcontextprotocol/server-filesystem ."
|
||||
|
||||
# Generate Cursor / Claude Desktop config
|
||||
praisonai mcp config-generate --client cursor --transport stdio</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Deep dive: <a href="https://mer.vin/2026/07/praisonai-mcp-praisonai-mcp-server-host-client-config-in-depth/">praisonai-mcp in-depth</a></p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 6 — Browser automation (via browser tier)</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>pip install "praisonai[browser]"
|
||||
|
||||
# Extension bridge server + LLM browser agent
|
||||
praisonai browser start
|
||||
praisonai browser run "Find the latest Python release notes"
|
||||
|
||||
# Diagnostics
|
||||
praisonai browser doctor
|
||||
|
||||
# Lightweight tool shim (code tier — no praisonai-browser required)
|
||||
praisonai browser-tool snapshot https://example.com
|
||||
|
||||
# Standalone script
|
||||
praisonai-browser start
|
||||
praisonai-browser run "Click the login button"</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Three layers: agents protocols → <code>praisonai browser-tool</code> (praisonai-tools) → <code>praisonai browser</code> (extension bridge, CDP, Playwright).</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Deep dive: <a href="https://mer.vin/2026/07/praisonai-browser-praisonai-browser-browser-automation-in-depth/">praisonai-browser in-depth</a></p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 7 — Dashboard stack</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>pip install "praisonai[claw,flow,ui]"
|
||||
|
||||
# Individual services
|
||||
praisonai claw # :8082 — ops dashboard
|
||||
praisonai flow # :7861 — Langflow
|
||||
praisonai ui # :8081 — clean chat
|
||||
|
||||
# Unified launcher (wrapper-only)
|
||||
praisonai dashboard # :3000
|
||||
praisonai dashboard --port 9000 --no-auto-start</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Port</th><th>Service</th></tr></thead><tbody><tr><td>3000</td><td>Dashboard hub</td></tr><tr><td>7860/7861</td><td>Langflow</td></tr><tr><td>8081</td><td>UI chat</td></tr><tr><td>8082</td><td>Claw</td></tr><tr><td>8765</td><td>Gateway WebSocket</td></tr><tr><td>8005</td><td>Jobs API</td></tr><tr><td>8080</td><td>MCP HTTP stream (optional)</td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 8 — Multi-framework YAML</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>pip install "praisonai[crewai]"
|
||||
export OPENAI_API_KEY=sk-...
|
||||
|
||||
praisonai --framework crewai --auto "Create a movie script"
|
||||
praisonai run --file agents.yaml --framework crewai
|
||||
|
||||
pip install "praisonai[autogen]"
|
||||
praisonai --framework autogen agents.yaml</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Protocols live in <code>praisonaiagents/frameworks/</code>; adapter implementations in wrapper <code>framework_adapters/</code>. Entry-point group: <code>praisonai.framework_adapters</code>.</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 9 — Scheduler (24/7 agents)</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>praisonai schedule add --agent my_agent.yaml --expr "hourly"
|
||||
praisonai schedule list
|
||||
praisonai schedule start</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code># Python
|
||||
from praisonai.scheduler import AgentScheduler
|
||||
from praisonaiagents import Agent
|
||||
|
||||
agent = Agent(name="NewsChecker", instructions="Check latest AI news")
|
||||
scheduler = AgentScheduler(agent=agent, task="Summarise top 3 AI stories")
|
||||
scheduler.start("hourly", run_immediately=True)</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Three scheduler layers: job store (<code>praisonaiagents.scheduler</code>) → CLI (<code>praisonai schedule</code>, wrapper) → gateway tick delivery (<code>praisonai_bot.scheduler.executor</code>).</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 10 — Jobs API & wrapper-only surfaces</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Jobs, recipe, serve, sandbox, and deploy remain wrapper-owned today (deferred or C13 candidates).</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code># Jobs API (wrapper — port 8005)
|
||||
python -m uvicorn praisonai.jobs.server:create_app --port 8005 --factory
|
||||
praisonai run submit "Analyse AI trends"
|
||||
praisonai run submit "Analyse news" --recipe news-analyzer --wait
|
||||
export PRAISONAI_JOBS_API_KEY=your-secret
|
||||
|
||||
# HTTP serve (wrapper orchestration — Typer shell in code tier)
|
||||
praisonai serve agents --port 8080
|
||||
praisonai serve recipe support-reply
|
||||
|
||||
# Sandbox / deploy (wrapper today — C13 extraction candidates)
|
||||
praisonai sandbox run "print hello" --backend docker
|
||||
praisonai deploy agents.yaml --provider aws</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 11 — Lazy CLI routing (six resident registries)</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Registry</th><th>Count</th><th>Examples</th><th>Loaded from</th><th>Gate</th></tr></thead><tbody><tr><td>Code-resident</td><td>~40</td><td>run, chat, code, daemon, serve, config</td><td><code>praisonai_code.cli.commands.*</code></td><td>always</td></tr><tr><td>Bot-resident</td><td>8</td><td>bot, gateway, onboard, kanban, claw</td><td><code>praisonai_bot.cli.commands.*</code></td><td><code>bot_package_available()</code></td></tr><tr><td>Train-resident</td><td>1</td><td>train</td><td><code>praisonai_train.cli.commands.*</code></td><td><code>train_package_available()</code></td></tr><tr><td>Browser-resident</td><td>1</td><td>browser</td><td><code>praisonai_browser.cli.commands.*</code></td><td><code>browser_package_available()</code></td></tr><tr><td>MCP-resident</td><td>1</td><td>mcp</td><td><code>praisonai_mcp.cli.commands.*</code></td><td><code>mcp_package_available()</code></td></tr><tr><td>Wrapper-resident</td><td>22</td><td>dashboard, schedule, flow, langfuse, recipe</td><td><code>praisonai.cli.commands.*</code></td><td><code>wrapper_available()</code></td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>praisonai --help # 70+ commands, lazy-loaded
|
||||
praisonai doctor
|
||||
praisonai train --help
|
||||
praisonai mcp --help
|
||||
praisonai browser --help
|
||||
praisonai dashboard --help</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Standalone <code>praisonai-code</code> hides bot/train/browser/mcp/wrapper commands from <code>--help</code> when those packages are absent.</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 12 — Shim map (backward compat)</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Old import / path</th><th>Canonical target</th></tr></thead><tbody><tr><td><code>praisonai.cli.main</code></td><td><code>praisonai_code.cli.main</code></td></tr><tr><td><code>praisonai.runtime.*</code></td><td><code>praisonai_code.runtime.*</code></td></tr><tr><td><code>praisonai.bots.*</code></td><td><code>praisonai_bot.bots.*</code></td></tr><tr><td><code>praisonai.gateway.*</code></td><td><code>praisonai_bot.gateway.*</code></td></tr><tr><td><code>praisonai.train.*</code></td><td><code>praisonai_train.train.*</code></td></tr><tr><td><code>praisonai.browser.*</code></td><td><code>praisonai_browser.*</code></td></tr><tr><td><code>praisonai.mcp_server.*</code></td><td><code>praisonai_mcp.mcp_server.*</code></td></tr><tr><td><code>praisonai.scheduler.executor</code></td><td><code>praisonai_bot.scheduler.executor</code></td></tr><tr><td><code>setup-conda-env</code></td><td><code>praisonai_train.setup.setup_conda_env</code></td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code># These must keep working
|
||||
from praisonai.cli.main import PraisonAI
|
||||
from praisonai.bots import Bot
|
||||
from praisonai.gateway import WebSocketGateway
|
||||
from praisonai.train.agents import AgentTrainer
|
||||
from praisonai.mcp_server import MCPServer
|
||||
python -m praisonai run "hello"
|
||||
python -m praisonai.runtime</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Section 13 — Bridges (no PyPI cycles)</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Bridge</th><th>Direction</th><th>Use</th></tr></thead><tbody><tr><td><code>_wrapper_bridge</code></td><td>code → wrapper</td><td>serve handlers, frameworks, jobs</td></tr><tr><td><code>_bot_bridge</code></td><td>code → bot</td><td>bot/gateway CLI routing</td></tr><tr><td><code>_train_bridge</code></td><td>code → train</td><td>train CLI routing</td></tr><tr><td><code>_browser_bridge</code></td><td>code → browser</td><td>browser CLI routing</td></tr><tr><td><code>_mcp_bridge</code></td><td>code → mcp</td><td>mcp CLI routing</td></tr><tr><td><code>_code_bridge</code></td><td>tier-2 → code</td><td>config kernel, legacy dispatch</td></tr><tr><td><code>_wrapper_bridge</code></td><td>tier-2 → wrapper</td><td>capabilities, recipe, jobs/UI</td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">When to use wrapper vs a Tier-2 package alone</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Your goal</th><th>Install</th></tr></thead><tbody><tr><td>Only terminal agents</td><td><code>praisonai-code</code> (+ optional <code>praisonaiagents</code>)</td></tr><tr><td>Only bots on Telegram/Slack</td><td><code>praisonai-bot</code></td></tr><tr><td>Only train / fine-tune</td><td><code>praisonai-train</code></td></tr><tr><td>Only automate browser</td><td><code>praisonai-browser</code></td></tr><tr><td>Only MCP server for IDE</td><td><code>praisonai-mcp</code></td></tr><tr><td>Several of the above + serve / dashboard / framework YAML</td><td><code>pip install praisonai</code></td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">What stays in wrapper (not extracted)</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Surface</th><th>Why it stays</th></tr></thead><tbody><tr><td><code>dashboard</code> / <code>flow</code> / <code>ui</code></td><td>Langflow extra + launcher glue — not a standalone PyPI product</td></tr><tr><td><code>claw</code></td><td>Default app lives in <code>praisonai-bot</code>; wrapper is a thin shim</td></tr><tr><td><code>framework_adapters/</code></td><td>Thin registry; heavy impls in external PraisonAI-Frameworks</td></tr><tr><td><code>schedule</code> CLI</td><td>Three-layer split: agents store + wrapper CLI + bot executor</td></tr><tr><td><code>serve</code> / <code>recipe</code> / <code>jobs</code></td><td>~14k LOC deferred epic — high coupling (C15)</td></tr><tr><td><code>sandbox</code> / <code>deploy</code></td><td>Still in wrapper today; C13 extraction candidates</td></tr><tr><td><code>standardise</code> / suite runner</td><td>Maintainer/docs tooling — internal only</td></tr><tr><td><code>persist</code> / enterprise adapters</td><td><strong>Rejected</strong> — agents already ship File/SQLite defaults; no crisp standalone story</td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Roadmap — C13+ candidates</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Package</th><th>Status</th><th>Choose when…</th><th>Standalone install (planned)</th></tr></thead><tbody><tr><td><code>praisonai-sandbox</code></td><td><strong>C13 preferred</strong> (execution goal)</td><td>Isolated agent code execution — Docker/E2B/Modal/Daytona/Sandlock</td><td><code>pip install praisonai-sandbox[docker]</code></td></tr><tr><td><code>praisonai-deploy</code></td><td><strong>C13 alt</strong> (deploy goal)</td><td>Ship <code>agents.yaml</code> to Docker/AWS/GCP/Azure</td><td><code>pip install praisonai-deploy</code></td></tr><tr><td><code>praisonai-replay</code></td><td>Conditional C14+</td><td>Deterministic replay from run ledger / traces</td><td>After product sign-off</td></tr><tr><td><code>praisonai-serve</code></td><td>Deferred C15</td><td>HTTP API hosting without full umbrella (~14k LOC)</td><td>With recipe/jobs decision</td></tr><tr><td><code>praisonai-recipe</code></td><td>Deferred</td><td>NL → agent YAML without jobs UI</td><td>Tied to jobs + serve</td></tr><tr><td><code>praisonai-jobs</code></td><td>Deferred</td><td>Async long-running agent jobs API</td><td>Extract with serve or stay wrapper</td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>C13 decision rule: <strong>sandbox</strong> = where code runs; <strong>deploy</strong> = where the agent service lives. Ship one first after product sign-off.</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Standalone extraction litmus test</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Criterion</th><th>Pass (browser/MCP)</th><th>Fail (example)</th></tr></thead><tbody><tr><td>Named user goal without full <code>praisonai</code></td><td><code>pip install praisonai-browser</code> → automate Chrome</td><td><code>pip install praisonai-persist</code> → ???</td></tr><tr><td>Coherent product boundary</td><td>One Typer group / one console script</td><td>Whole <code>cli/features/</code> orchestration layer</td></tr><tr><td>PyPI deps: <code>praisonaiagents</code> only</td><td>Tier-2 + lazy <code>_wrapper_bridge</code></td><td>Hard <code>import praisonai</code> at module level</td></tr><tr><td>Worth a separate install</td><td>User who never wants dashboard/serve/bots</td><td>Internal docs tooling</td></tr><tr><td>Extraction shape</td><td>C10 single PR (~3–11k LOC)</td><td>Mega-epic without product sign-off</td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">When to use which tier</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Scenario</th><th>Install</th></tr></thead><tbody><tr><td>Pure Python agents, minimal deps</td><td><code>pip install praisonaiagents</code></td></tr><tr><td>Terminal coding agent, CI, headless</td><td><code>pip install praisonai-code</code></td></tr><tr><td>Telegram/Discord gateway only</td><td><code>pip install "praisonai-bot[gateway,bot]"</code></td></tr><tr><td>Agent training (no GPU)</td><td><code>pip install praisonai-train</code></td></tr><tr><td>LLM fine-tuning (GPU)</td><td><code>pip install "praisonai-train[llm]"</code></td></tr><tr><td>MCP server for Cursor/Claude</td><td><code>pip install "praisonai-mcp[all]"</code></td></tr><tr><td>Browser automation (extension/CDP)</td><td><code>pip install "praisonai-browser[all]"</code></td></tr><tr><td>CrewAI/AutoGen YAML</td><td><code>pip install "praisonai[crewai]"</code></td></tr><tr><td>Full ops + channels + dashboard</td><td><code>pip install "praisonai[claw]"</code></td></tr><tr><td>Everything in one install</td><td><code>pip install praisonai</code></td></tr><tr><td>Library author / embedding</td><td><code>praisonaiagents</code> only</td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Seven-package publish order</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:code -->
|
||||
<pre class="wp-block-code"><code>praisonaiagents
|
||||
→ praisonai-code + praisonai-bot + praisonai-train + praisonai-browser + praisonai-mcp
|
||||
→ praisonai (wrapper pins all tier-2 deps last)</code></pre>
|
||||
<!-- /wp:code -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>CI: <code>.github/workflows/pypi-release.yml</code> · selective publish: <code>publish_all.py --changed-only</code> (default). Playbook per package: manifest → TDD compat tests → git mv → shims → <code>_X_RESIDENT_COMMANDS</code> → CI shard → PyPI.</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:heading -->
|
||||
<h2 class="wp-block-heading">Summary</h2>
|
||||
<!-- /wp:heading -->
|
||||
|
||||
<!-- wp:table -->
|
||||
<figure class="wp-block-table"><table><thead><tr><th>Item</th><th>Value</th></tr></thead><tbody><tr><td>Tier</td><td>3 (integration wrapper)</td></tr><tr><td>PyPI</td><td><code>praisonai</code> v4.6.150</td></tr><tr><td>Shipped tier-2 deps</td><td>code, bot, train, browser, mcp (C0–C12)</td></tr><tr><td>Resident registries</td><td>6 (code + bot + train + browser + mcp + wrapper)</td></tr><tr><td>Wrapper-only (~140k LOC)</td><td>serve, recipe, jobs, dashboard, sandbox, deploy, frameworks</td></tr><tr><td>Next extraction</td><td>C13: sandbox or deploy (product sign-off)</td></tr><tr><td>One-liner still works</td><td><code>pip install praisonai</code></td></tr></tbody></table></figure>
|
||||
<!-- /wp:table -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Architecture overview: Seven-package C7–C12 extraction · Post-C12 roadmap (C13+)</p>
|
||||
<!-- /wp:paragraph -->
|
||||
|
||||
<!-- wp:paragraph -->
|
||||
<p>Series: <a href="https://mer.vin/2026/07/praisonai-code-praisonai-code-terminal-cli-in-depth/">praisonai-code</a> · <a href="https://mer.vin/2026/07/praisonai-bot-praisonai-bot-gateway-channels-in-depth/">praisonai-bot</a> · <a href="https://mer.vin/2026/07/praisonai-train-praisonai-train-fine-tuning-agent-training-in-depth/">praisonai-train</a> · <a href="https://mer.vin/2026/07/praisonai-browser-praisonai-browser-browser-automation-in-depth/">praisonai-browser</a> · <a href="https://mer.vin/2026/07/praisonai-mcp-praisonai-mcp-server-host-client-config-in-depth/">praisonai-mcp</a> · praisonai wrapper</p>
|
||||
<!-- /wp:paragraph -->
|
||||
@@ -1,5 +1,5 @@
|
||||
name: Install PraisonAI monorepo packages
|
||||
description: Install praisonai-agents, praisonai-code, praisonai-bot, praisonai-train, praisonai-browser, praisonai-mcp, and praisonai wrapper from local paths (mirrors PyPI seven-package install order).
|
||||
description: Install praisonai-agents, praisonai-code, praisonai-bot, praisonai-train, praisonai-browser, praisonai-mcp, praisonai-sandbox, and praisonai wrapper from local paths (mirrors PyPI eight-package install order).
|
||||
|
||||
inputs:
|
||||
repo-root:
|
||||
@@ -39,6 +39,7 @@ runs:
|
||||
TRAIN_PKG="${BASE}src/praisonai-train"
|
||||
BROWSER_PKG="${BASE}src/praisonai-browser"
|
||||
MCP_PKG="${BASE}src/praisonai-mcp"
|
||||
SANDBOX_PKG="${BASE}src/praisonai-sandbox"
|
||||
WRAPPER_PKG="${BASE}src/praisonai"
|
||||
|
||||
normalise_extras() {
|
||||
@@ -66,6 +67,8 @@ runs:
|
||||
uv pip install --system -e .
|
||||
cd "$MCP_PKG"
|
||||
uv pip install --system -e .
|
||||
cd "$SANDBOX_PKG"
|
||||
uv pip install --system -e .
|
||||
cd "$WRAPPER_PKG"
|
||||
uv pip install --system "${WRAPPER_SUFFIX}"
|
||||
else
|
||||
@@ -81,6 +84,8 @@ runs:
|
||||
pip install -e .
|
||||
cd "$MCP_PKG"
|
||||
pip install -e .
|
||||
cd "$SANDBOX_PKG"
|
||||
pip install -e .
|
||||
cd "$WRAPPER_PKG"
|
||||
pip install -e "${WRAPPER_SUFFIX}"
|
||||
fi
|
||||
@@ -40,6 +40,62 @@ assert('claude final reply detected', mg.isClaudeFinalReplyComment(withClaudeRep
|
||||
|
||||
assert('cancelled detect-and-trigger does not block', mg.OPTIONAL_CANCELLED_CHECKS.has('detect-and-trigger'));
|
||||
|
||||
const coreGreenRuns = [
|
||||
{ name: 'test-core', status: 'completed', conclusion: 'success' },
|
||||
{ name: 'test-core (cli)', status: 'completed', conclusion: 'success' },
|
||||
{ name: 'smoke', status: 'completed', conclusion: 'cancelled' },
|
||||
{ name: 'test-windows', status: 'completed', conclusion: 'cancelled' },
|
||||
];
|
||||
assert('core green allows cancelled smoke', mg.isAcceptableCheckConclusion(
|
||||
{ name: 'smoke', status: 'completed', conclusion: 'cancelled' },
|
||||
coreGreenRuns
|
||||
));
|
||||
assert('core green allows cancelled test-windows', mg.isAcceptableCheckConclusion(
|
||||
{ name: 'test-windows', status: 'completed', conclusion: 'cancelled' },
|
||||
coreGreenRuns
|
||||
));
|
||||
assert('cancelled smoke blocks when core missing', !mg.isAcceptableCheckConclusion(
|
||||
{ name: 'smoke', status: 'completed', conclusion: 'cancelled' },
|
||||
[{ name: 'smoke', status: 'completed', conclusion: 'cancelled' }]
|
||||
));
|
||||
assert('cancelled smoke blocks when core failed', !mg.isAcceptableCheckConclusion(
|
||||
{ name: 'smoke', status: 'completed', conclusion: 'cancelled' },
|
||||
[
|
||||
{ name: 'test-core', status: 'completed', conclusion: 'failure' },
|
||||
{ name: 'smoke', status: 'completed', conclusion: 'cancelled' },
|
||||
]
|
||||
));
|
||||
assert('bestRunsByName prefers success over cancelled', (() => {
|
||||
const best = mg.bestRunsByName([
|
||||
{ name: 'smoke', conclusion: 'cancelled' },
|
||||
{ name: 'smoke', conclusion: 'success' },
|
||||
]);
|
||||
return best.length === 1 && best[0].conclusion === 'success';
|
||||
})());
|
||||
assert('bestRunsByName prefers pending over completed cancelled', (() => {
|
||||
const best = mg.bestRunsByName([
|
||||
{ name: 'smoke', status: 'completed', conclusion: 'cancelled' },
|
||||
{ name: 'smoke', status: 'in_progress', conclusion: null },
|
||||
]);
|
||||
return best.length === 1 && best[0].status === 'in_progress';
|
||||
})());
|
||||
assert('bestRunsByName keeps pending regardless of order', (() => {
|
||||
const best = mg.bestRunsByName([
|
||||
{ name: 'smoke', status: 'in_progress', conclusion: null },
|
||||
{ name: 'smoke', status: 'completed', conclusion: 'success' },
|
||||
]);
|
||||
return best.length === 1 && best[0].status === 'in_progress';
|
||||
})());
|
||||
assert('allChecksGreenOnSha style: pending re-run blocks despite core green', (() => {
|
||||
const runs = mg.bestRunsByName([
|
||||
{ name: 'test-core', status: 'completed', conclusion: 'success' },
|
||||
{ name: 'smoke', status: 'completed', conclusion: 'cancelled' },
|
||||
{ name: 'smoke', status: 'queued', conclusion: null },
|
||||
]);
|
||||
const smoke = runs.find((r) => r.name === 'smoke');
|
||||
return smoke.status !== 'completed';
|
||||
})());
|
||||
|
||||
// Stale-FINAL recovery guards (PR #2560 push loop)
|
||||
const nowMs = Date.now();
|
||||
const iso = (ms) => new Date(ms).toISOString();
|
||||
|
||||
@@ -44,6 +44,8 @@ const BLOCK_LABELS = new Set([
|
||||
const MERGE_READY_LABEL = 'pipeline/merge-ready';
|
||||
/** Superseded concurrency runs; must not block merge when real tests passed. */
|
||||
const OPTIONAL_CANCELLED_CHECKS = new Set(['detect-and-trigger']);
|
||||
/** Cancelled smoke/windows after timeout are non-blocking when core shards passed on HEAD. */
|
||||
const OPTIONAL_CANCELLED_WHEN_CORE_GREEN = new Set(['smoke', 'test-windows']);
|
||||
const BOT_REVIEWER_PATTERNS = [
|
||||
'coderabbit',
|
||||
'qodo',
|
||||
@@ -325,8 +327,70 @@ function listFailedChecksOnSha(runs) {
|
||||
});
|
||||
}
|
||||
|
||||
function isCoreTestRun(run) {
|
||||
const name = run?.name || '';
|
||||
return name === 'test-core' || name.startsWith('test-core ') || name === 'test-core-collect';
|
||||
}
|
||||
|
||||
function coreTestsGreenOnRuns(runs) {
|
||||
const coreRuns = (runs || []).filter(isCoreTestRun);
|
||||
if (coreRuns.length === 0) return false;
|
||||
return coreRuns.every(
|
||||
(run) =>
|
||||
run.status === 'completed' &&
|
||||
['success', 'neutral', 'skipped'].includes(run.conclusion)
|
||||
);
|
||||
}
|
||||
|
||||
function checkConclusionRank(conclusion) {
|
||||
if (conclusion === 'success') return 4;
|
||||
if (conclusion === 'neutral') return 3;
|
||||
if (conclusion === 'skipped') return 2;
|
||||
if (conclusion === 'cancelled') return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
function isPendingRun(run) {
|
||||
return !!(run && run.status && run.status !== 'completed');
|
||||
}
|
||||
|
||||
function bestRunsByName(runs) {
|
||||
const byName = new Map();
|
||||
for (const run of runs || []) {
|
||||
if (!run) continue;
|
||||
const existing = byName.get(run.name);
|
||||
if (!existing) {
|
||||
byName.set(run.name, run);
|
||||
continue;
|
||||
}
|
||||
const runPending = isPendingRun(run);
|
||||
const existingPending = isPendingRun(existing);
|
||||
if (existingPending) continue;
|
||||
if (
|
||||
runPending ||
|
||||
checkConclusionRank(run.conclusion) > checkConclusionRank(existing.conclusion)
|
||||
) {
|
||||
byName.set(run.name, run);
|
||||
}
|
||||
}
|
||||
return [...byName.values()];
|
||||
}
|
||||
|
||||
function isAcceptableCheckConclusion(run, runs) {
|
||||
if (['success', 'neutral', 'skipped'].includes(run.conclusion)) return true;
|
||||
if (run.conclusion === 'cancelled' && OPTIONAL_CANCELLED_CHECKS.has(run.name)) return true;
|
||||
if (
|
||||
run.conclusion === 'cancelled' &&
|
||||
OPTIONAL_CANCELLED_WHEN_CORE_GREEN.has(run.name) &&
|
||||
coreTestsGreenOnRuns(runs)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async function allChecksGreenOnSha(github, owner, repo, sha, core) {
|
||||
const runs = await listChecksOnSha(github, owner, repo, sha);
|
||||
const runs = bestRunsByName(await listChecksOnSha(github, owner, repo, sha));
|
||||
if (runs.length === 0) {
|
||||
core?.info?.(`No check runs on ${sha.slice(0, 7)} — allowing (e.g. docs-only PR)`);
|
||||
return true;
|
||||
@@ -336,13 +400,17 @@ async function allChecksGreenOnSha(github, owner, repo, sha, core) {
|
||||
core?.info?.(`Check pending: ${run.name} (${run.status})`);
|
||||
return false;
|
||||
}
|
||||
const ok =
|
||||
['success', 'neutral', 'skipped'].includes(run.conclusion) ||
|
||||
(run.conclusion === 'cancelled' && OPTIONAL_CANCELLED_CHECKS.has(run.name));
|
||||
if (!ok) {
|
||||
if (!isAcceptableCheckConclusion(run, runs)) {
|
||||
core?.info?.(`Check failed: ${run.name} (${run.conclusion})`);
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
run.conclusion === 'cancelled' &&
|
||||
OPTIONAL_CANCELLED_WHEN_CORE_GREEN.has(run.name) &&
|
||||
coreTestsGreenOnRuns(runs)
|
||||
) {
|
||||
core?.info?.(`Ignoring cancelled ${run.name} — test-core green on HEAD`);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -855,6 +923,12 @@ module.exports = {
|
||||
finalClaudeCompletedOnSha,
|
||||
getMergeState,
|
||||
OPTIONAL_CANCELLED_CHECKS,
|
||||
OPTIONAL_CANCELLED_WHEN_CORE_GREEN,
|
||||
isCoreTestRun,
|
||||
coreTestsGreenOnRuns,
|
||||
isPendingRun,
|
||||
bestRunsByName,
|
||||
isAcceptableCheckConclusion,
|
||||
listChecksOnSha,
|
||||
listFailedChecksOnSha,
|
||||
allChecksGreenOnSha,
|
||||
|
||||
@@ -52,9 +52,11 @@ jobs:
|
||||
// Kind labels
|
||||
if (content.includes('feature request') || content.includes('[feature')) {
|
||||
labels.push('enhancement');
|
||||
} else if (title.includes('[question]') || content.includes('how do i') || content.includes('how to')) {
|
||||
labels.push('question');
|
||||
} else if (content.includes('bug') || content.includes('error') || content.includes('crash') || content.includes('traceback')) {
|
||||
labels.push('bug');
|
||||
} else if (content.includes('question') || content.includes('how do i') || content.includes('how to')) {
|
||||
} else if (content.includes('question')) {
|
||||
labels.push('question');
|
||||
}
|
||||
|
||||
@@ -105,8 +107,8 @@ jobs:
|
||||
|
||||
// --- Auto-trigger Claude for owner issues and actionable external bug/enhancement reports ---
|
||||
const titleLower = (issue.title || '').toLowerCase();
|
||||
const hasActionableTitle = /^(bug:|fix[:(]|feat:|ux:|enhancement:)/.test(titleLower);
|
||||
const autoClaude = isOwner || (hasActionableTitle && (labels.includes('bug') || labels.includes('enhancement')));
|
||||
const hasActionableTitle = /^(\[[^\]]+\]|bug:|fix[:(]|feat:|ux:|enhancement:)/i.test(titleLower);
|
||||
const autoClaude = isOwner || (hasActionableTitle && (labels.includes('bug') || labels.includes('enhancement')) && !labels.includes('question'));
|
||||
if (autoClaude) {
|
||||
await github.rest.issues.addLabels({
|
||||
issue_number: issue.number,
|
||||
|
||||
@@ -56,6 +56,11 @@ on:
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
skip_sandbox:
|
||||
description: 'Skip praisonai-sandbox publish (recovery if already on PyPI)'
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
agents_version:
|
||||
description: 'Override agents version (empty = auto bump from pyproject.toml)'
|
||||
required: false
|
||||
@@ -91,6 +96,11 @@ on:
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
sandbox_version:
|
||||
description: 'Override praisonai-sandbox version (empty = auto bump from pyproject.toml)'
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
source:
|
||||
description: 'Trigger source (audit trail)'
|
||||
required: false
|
||||
@@ -167,12 +177,14 @@ jobs:
|
||||
TRAIN_OVERRIDE: ${{ inputs.train_version }}
|
||||
BROWSER_OVERRIDE: ${{ inputs.browser_version }}
|
||||
MCP_OVERRIDE: ${{ inputs.mcp_version }}
|
||||
SANDBOX_OVERRIDE: ${{ inputs.sandbox_version }}
|
||||
SKIP_AGENTS: ${{ inputs.skip_agents }}
|
||||
SKIP_CODE: ${{ inputs.skip_code }}
|
||||
SKIP_BOT: ${{ inputs.skip_bot }}
|
||||
SKIP_TRAIN: ${{ inputs.skip_train }}
|
||||
SKIP_BROWSER: ${{ inputs.skip_browser }}
|
||||
SKIP_MCP: ${{ inputs.skip_mcp }}
|
||||
SKIP_SANDBOX: ${{ inputs.skip_sandbox }}
|
||||
run: |
|
||||
python <<'PY'
|
||||
import os
|
||||
@@ -208,6 +220,7 @@ jobs:
|
||||
train_override = os.environ.get("TRAIN_OVERRIDE", "").strip()
|
||||
browser_override = os.environ.get("BROWSER_OVERRIDE", "").strip()
|
||||
mcp_override = os.environ.get("MCP_OVERRIDE", "").strip()
|
||||
sandbox_override = os.environ.get("SANDBOX_OVERRIDE", "").strip()
|
||||
validate_version("agents_version", agents_override)
|
||||
validate_version("wrapper_version", wrapper_override)
|
||||
validate_version("code_version", code_override)
|
||||
@@ -215,6 +228,7 @@ jobs:
|
||||
validate_version("train_version", train_override)
|
||||
validate_version("browser_version", browser_override)
|
||||
validate_version("mcp_version", mcp_override)
|
||||
validate_version("sandbox_version", sandbox_override)
|
||||
|
||||
agents_pyproject = root / "src/praisonai-agents/pyproject.toml"
|
||||
agents_content = agents_pyproject.read_text()
|
||||
@@ -264,6 +278,14 @@ jobs:
|
||||
sys.exit(1)
|
||||
current_mcp = mcp_match.group(1)
|
||||
|
||||
sandbox_pyproject = root / "src/praisonai-sandbox/pyproject.toml"
|
||||
sandbox_content = sandbox_pyproject.read_text()
|
||||
sandbox_match = re.search(r'^version\s*=\s*"([^"]+)"', sandbox_content, re.MULTILINE)
|
||||
if not sandbox_match:
|
||||
print("Could not read praisonai-sandbox version from pyproject.toml", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
current_sandbox = sandbox_match.group(1)
|
||||
|
||||
wrapper_version_py = root / "src/praisonai/praisonai/version.py"
|
||||
wrapper_content = wrapper_version_py.read_text()
|
||||
wrapper_match = re.search(r'__version__ = "([^"]+)"', wrapper_content)
|
||||
@@ -279,6 +301,7 @@ jobs:
|
||||
("train", current_train),
|
||||
("browser", current_browser),
|
||||
("mcp", current_mcp),
|
||||
("sandbox", current_sandbox),
|
||||
("wrapper", current_wrapper),
|
||||
):
|
||||
if not SEMVER.match(value):
|
||||
@@ -291,6 +314,7 @@ jobs:
|
||||
skip_train = os.environ.get("SKIP_TRAIN", "false").lower() == "true"
|
||||
skip_browser = os.environ.get("SKIP_BROWSER", "false").lower() == "true"
|
||||
skip_mcp = os.environ.get("SKIP_MCP", "false").lower() == "true"
|
||||
skip_sandbox = os.environ.get("SKIP_SANDBOX", "false").lower() == "true"
|
||||
|
||||
if agents_override:
|
||||
agents_version = agents_override
|
||||
@@ -334,6 +358,13 @@ jobs:
|
||||
else:
|
||||
mcp_version = bump_version(current_mcp)
|
||||
|
||||
if sandbox_override:
|
||||
sandbox_version = sandbox_override
|
||||
elif skip_sandbox:
|
||||
sandbox_version = current_sandbox
|
||||
else:
|
||||
sandbox_version = bump_version(current_sandbox)
|
||||
|
||||
if wrapper_override:
|
||||
wrapper_version = wrapper_override
|
||||
else:
|
||||
@@ -347,6 +378,7 @@ jobs:
|
||||
fh.write(f"train_version={train_version}\n")
|
||||
fh.write(f"browser_version={browser_version}\n")
|
||||
fh.write(f"mcp_version={mcp_version}\n")
|
||||
fh.write(f"sandbox_version={sandbox_version}\n")
|
||||
fh.write(f"wrapper_version={wrapper_version}\n")
|
||||
fh.write(f"current_agents={current_agents}\n")
|
||||
fh.write(f"current_code={current_code}\n")
|
||||
@@ -354,6 +386,7 @@ jobs:
|
||||
fh.write(f"current_train={current_train}\n")
|
||||
fh.write(f"current_browser={current_browser}\n")
|
||||
fh.write(f"current_mcp={current_mcp}\n")
|
||||
fh.write(f"current_sandbox={current_sandbox}\n")
|
||||
fh.write(f"current_wrapper={current_wrapper}\n")
|
||||
|
||||
print(f"Agents: {current_agents} -> {agents_version}")
|
||||
@@ -362,6 +395,7 @@ jobs:
|
||||
print(f"Train: {current_train} -> {train_version}")
|
||||
print(f"Browser: {current_browser} -> {browser_version}")
|
||||
print(f"MCP: {current_mcp} -> {mcp_version}")
|
||||
print(f"Sandbox: {current_sandbox} -> {sandbox_version}")
|
||||
print(f"Wrapper: {current_wrapper} -> {wrapper_version}")
|
||||
PY
|
||||
|
||||
@@ -375,6 +409,7 @@ jobs:
|
||||
TRAIN_VERSION: ${{ steps.versions.outputs.train_version }}
|
||||
BROWSER_VERSION: ${{ steps.versions.outputs.browser_version }}
|
||||
MCP_VERSION: ${{ steps.versions.outputs.mcp_version }}
|
||||
SANDBOX_VERSION: ${{ steps.versions.outputs.sandbox_version }}
|
||||
WRAPPER_VERSION: ${{ steps.versions.outputs.wrapper_version }}
|
||||
INPUT_SKIP_AGENTS: ${{ inputs.skip_agents }}
|
||||
INPUT_SKIP_CODE: ${{ inputs.skip_code }}
|
||||
@@ -382,6 +417,7 @@ jobs:
|
||||
INPUT_SKIP_TRAIN: ${{ inputs.skip_train }}
|
||||
INPUT_SKIP_BROWSER: ${{ inputs.skip_browser }}
|
||||
INPUT_SKIP_MCP: ${{ inputs.skip_mcp }}
|
||||
INPUT_SKIP_SANDBOX: ${{ inputs.skip_sandbox }}
|
||||
INPUT_SKIP_WRAPPER: ${{ inputs.skip_wrapper }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -391,6 +427,7 @@ jobs:
|
||||
skip_train="${INPUT_SKIP_TRAIN}"
|
||||
skip_browser="${INPUT_SKIP_BROWSER}"
|
||||
skip_mcp="${INPUT_SKIP_MCP}"
|
||||
skip_sandbox="${INPUT_SKIP_SANDBOX}"
|
||||
skip_wrapper="${INPUT_SKIP_WRAPPER}"
|
||||
if [ "$skip_agents" != "true" ]; then
|
||||
curl -fsSL "https://pypi.org/pypi/praisonaiagents/${AGENTS_VERSION}/json" >/dev/null && skip_agents=true || true
|
||||
@@ -410,6 +447,9 @@ jobs:
|
||||
if [ "$skip_mcp" != "true" ]; then
|
||||
curl -fsSL "https://pypi.org/pypi/praisonai-mcp/${MCP_VERSION}/json" >/dev/null && skip_mcp=true || true
|
||||
fi
|
||||
if [ "$skip_sandbox" != "true" ]; then
|
||||
curl -fsSL "https://pypi.org/pypi/praisonai-sandbox/${SANDBOX_VERSION}/json" >/dev/null && skip_sandbox=true || true
|
||||
fi
|
||||
if [ "$skip_wrapper" != "true" ]; then
|
||||
curl -fsSL "https://pypi.org/pypi/praisonai/${WRAPPER_VERSION}/json" >/dev/null && skip_wrapper=true || true
|
||||
fi
|
||||
@@ -419,8 +459,9 @@ jobs:
|
||||
echo "skip_train=${skip_train}" >> "$GITHUB_OUTPUT"
|
||||
echo "skip_browser=${skip_browser}" >> "$GITHUB_OUTPUT"
|
||||
echo "skip_mcp=${skip_mcp}" >> "$GITHUB_OUTPUT"
|
||||
echo "skip_sandbox=${skip_sandbox}" >> "$GITHUB_OUTPUT"
|
||||
echo "skip_wrapper=${skip_wrapper}" >> "$GITHUB_OUTPUT"
|
||||
if [ "$skip_agents" = "true" ] && [ "$skip_code" = "true" ] && [ "$skip_bot" = "true" ] && [ "$skip_train" = "true" ] && [ "$skip_browser" = "true" ] && [ "$skip_mcp" = "true" ] && [ "$skip_wrapper" = "true" ]; then
|
||||
if [ "$skip_agents" = "true" ] && [ "$skip_code" = "true" ] && [ "$skip_bot" = "true" ] && [ "$skip_train" = "true" ] && [ "$skip_browser" = "true" ] && [ "$skip_mcp" = "true" ] && [ "$skip_sandbox" = "true" ] && [ "$skip_wrapper" = "true" ]; then
|
||||
echo "All target versions already on PyPI — nothing to publish."
|
||||
fi
|
||||
|
||||
@@ -433,6 +474,7 @@ jobs:
|
||||
steps.pypi_exists.outputs.skip_train == 'true' &&
|
||||
steps.pypi_exists.outputs.skip_browser == 'true' &&
|
||||
steps.pypi_exists.outputs.skip_mcp == 'true' &&
|
||||
steps.pypi_exists.outputs.skip_sandbox == 'true' &&
|
||||
steps.pypi_exists.outputs.skip_wrapper == 'true'
|
||||
run: echo "Idempotent skip — target versions already on PyPI."
|
||||
|
||||
@@ -887,6 +929,76 @@ jobs:
|
||||
git pull --rebase origin main
|
||||
git push origin main
|
||||
|
||||
- name: Publish praisonai-sandbox
|
||||
if: >-
|
||||
inputs.dry_run != true &&
|
||||
inputs.skip_sandbox != true &&
|
||||
steps.pypi_exists.outputs.skip_sandbox != 'true'
|
||||
working-directory: src/praisonai-sandbox
|
||||
env:
|
||||
NEW_VERSION: ${{ steps.versions.outputs.sandbox_version }}
|
||||
CURRENT: ${{ steps.versions.outputs.current_sandbox }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python <<'PY'
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
new = os.environ["NEW_VERSION"]
|
||||
current = os.environ["CURRENT"]
|
||||
pyproject = Path("pyproject.toml")
|
||||
content = pyproject.read_text()
|
||||
pyproject.write_text(content.replace(f'version = "{current}"', f'version = "{new}"', 1))
|
||||
version_py = Path("praisonai_sandbox/_version.py")
|
||||
version_content = version_py.read_text()
|
||||
version_py.write_text(re.sub(r'__version__ = "[^"]+"', f'__version__ = "{new}"', version_content, count=1))
|
||||
PY
|
||||
|
||||
rm -rf dist
|
||||
uv lock
|
||||
uv build
|
||||
uv publish
|
||||
|
||||
- name: Wait for praisonai-sandbox on PyPI
|
||||
if: >-
|
||||
inputs.dry_run != true &&
|
||||
inputs.skip_sandbox != true &&
|
||||
steps.pypi_exists.outputs.skip_sandbox != 'true'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PACKAGE="praisonai-sandbox"
|
||||
VERSION="${{ steps.versions.outputs.sandbox_version }}"
|
||||
MAX_WAIT=600
|
||||
INTERVAL=30
|
||||
START=$(date +%s)
|
||||
|
||||
while true; do
|
||||
if curl -fsSL "https://pypi.org/pypi/${PACKAGE}/${VERSION}/json" >/dev/null; then
|
||||
echo "✅ ${PACKAGE}==${VERSION} is available on PyPI"
|
||||
exit 0
|
||||
fi
|
||||
ELAPSED=$(( $(date +%s) - START ))
|
||||
if [ "$ELAPSED" -ge "$MAX_WAIT" ]; then
|
||||
echo "❌ Timeout waiting for ${PACKAGE}==${VERSION} on PyPI"
|
||||
exit 1
|
||||
fi
|
||||
echo "⏳ Waiting for ${PACKAGE}==${VERSION} ($(( MAX_WAIT - ELAPSED ))s remaining...)"
|
||||
sleep "$INTERVAL"
|
||||
done
|
||||
|
||||
- name: Commit praisonai-sandbox version bump
|
||||
if: >-
|
||||
inputs.dry_run != true &&
|
||||
inputs.skip_sandbox != true &&
|
||||
steps.pypi_exists.outputs.skip_sandbox != 'true'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git add src/praisonai-sandbox/pyproject.toml src/praisonai-sandbox/uv.lock src/praisonai-sandbox/praisonai_sandbox/_version.py
|
||||
git diff --cached --quiet && { echo "No sandbox files to commit"; exit 0; }
|
||||
git commit -m "Bump praisonai-sandbox to ${{ steps.versions.outputs.sandbox_version }}"
|
||||
git pull --rebase origin main
|
||||
git push origin main
|
||||
|
||||
- name: Bump and release wrapper
|
||||
if: >-
|
||||
inputs.dry_run != true &&
|
||||
@@ -901,6 +1013,7 @@ jobs:
|
||||
--train-pin "${{ steps.versions.outputs.train_version }}" \
|
||||
--browser-pin "${{ steps.versions.outputs.browser_version }}" \
|
||||
--mcp-pin "${{ steps.versions.outputs.mcp_version }}" \
|
||||
--sandbox-pin "${{ steps.versions.outputs.sandbox_version }}" \
|
||||
--wait --max-wait 600 --no-add-all
|
||||
|
||||
- name: Publish praisonai wrapper to PyPI
|
||||
@@ -948,6 +1061,7 @@ jobs:
|
||||
TRAIN="${{ steps.versions.outputs.train_version }}"
|
||||
BROWSER="${{ steps.versions.outputs.browser_version }}"
|
||||
MCP="${{ steps.versions.outputs.mcp_version }}"
|
||||
SANDBOX="${{ steps.versions.outputs.sandbox_version }}"
|
||||
WRAPPER="${{ steps.versions.outputs.wrapper_version }}"
|
||||
|
||||
if [ "${{ inputs.skip_agents }}" != "true" ]; then
|
||||
@@ -974,6 +1088,10 @@ jobs:
|
||||
echo "✅ praisonai-mcp==${MCP}"
|
||||
fi
|
||||
|
||||
if [ "${{ inputs.skip_sandbox }}" != "true" ]; then
|
||||
echo "✅ praisonai-sandbox==${SANDBOX}"
|
||||
fi
|
||||
|
||||
if [ "${{ inputs.skip_wrapper }}" != "true" ]; then
|
||||
echo "✅ praisonai==${WRAPPER}"
|
||||
fi
|
||||
@@ -0,0 +1,47 @@
|
||||
name: Terminal-Bench Smoke
|
||||
|
||||
# Manual (and optional weekly) smoke run of the `praisonai code` assistant on a
|
||||
# small verified Terminal-Bench 2.1 subset via the Harbor harness.
|
||||
# Never runs on PRs — it costs money (LLM calls) and is inherently flaky.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 6 * * 1" # Mondays 06:00 UTC
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
smoke:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install Harbor + PraisonAI
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install harbor
|
||||
pip install -e src/praisonai-agents -e src/praisonai
|
||||
|
||||
- name: Run Terminal-Bench 2.1 smoke subset
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
PYTHONPATH: .
|
||||
run: |
|
||||
harbor run -c examples/terminal_bench/job_code_smoke.yaml || true
|
||||
|
||||
- name: Upload Harbor results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: harbor-results
|
||||
path: |
|
||||
runs/
|
||||
**/results.json
|
||||
if-no-files-found: warn
|
||||
@@ -107,6 +107,13 @@ jobs:
|
||||
extra_ignore: ""
|
||||
pythonpath: >-
|
||||
${{ github.workspace }}/src/praisonai-agents
|
||||
- shard: sandbox
|
||||
workdir: src/praisonai-sandbox
|
||||
paths: >-
|
||||
tests/
|
||||
extra_ignore: ""
|
||||
pythonpath: >-
|
||||
${{ github.workspace }}/src/praisonai-agents
|
||||
- shard: subdirs
|
||||
paths: >-
|
||||
tests/unit/scheduler/
|
||||
@@ -173,6 +180,12 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
WORKDIR="${{ matrix.workdir || 'src/praisonai' }}"
|
||||
if [ "$WORKDIR" = "src/praisonai-sandbox" ]; then
|
||||
bash "$GITHUB_WORKSPACE/scripts/check_c13_sandbox_imports.sh"
|
||||
fi
|
||||
if [ "$WORKDIR" = "src/praisonai-mcp" ]; then
|
||||
bash "$GITHUB_WORKSPACE/scripts/check_c12_mcp_imports.sh"
|
||||
fi
|
||||
cd "$WORKDIR"
|
||||
if [ -n "${{ matrix.pythonpath || '' }}" ]; then
|
||||
export PYTHONPATH="${{ matrix.pythonpath }}:${PYTHONPATH:-}"
|
||||
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
# ============================================
|
||||
smoke:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 8
|
||||
timeout-minutes: 12
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
@@ -85,6 +85,7 @@ jobs:
|
||||
tests/unit/test_c9_1_boundaries.py \
|
||||
tests/unit/test_c11_browser_backward_compat.py \
|
||||
tests/unit/test_c12_mcp_backward_compat.py \
|
||||
tests/unit/test_c13_sandbox_backward_compat.py \
|
||||
-q --tb=line --timeout=30 --maxfail=5
|
||||
python -m pytest tests/unit/cli/ \
|
||||
../praisonai-bot/tests/unit/cli/test_onboard_command.py \
|
||||
@@ -251,10 +252,27 @@ jobs:
|
||||
bash "$GITHUB_WORKSPACE/scripts/check_c12_mcp_imports.sh"
|
||||
python -m pytest src/praisonai-mcp/tests/mcp_server -q -m "not network"
|
||||
|
||||
- name: Standalone praisonai-sandbox smoke (C13)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python -m venv /tmp/praisonai-sandbox-standalone
|
||||
/tmp/praisonai-sandbox-standalone/bin/pip install -q -e src/praisonai-agents -e src/praisonai-sandbox
|
||||
/tmp/praisonai-sandbox-standalone/bin/python -c "
|
||||
from praisonai_sandbox._version import __version__ as pkg_version
|
||||
import praisonai_sandbox
|
||||
assert praisonai_sandbox.__version__ == pkg_version
|
||||
for heavy in ('docker', 'modal', 'e2b'):
|
||||
assert heavy not in __import__('sys').modules
|
||||
print('sandbox standalone ok', praisonai_sandbox.__version__)
|
||||
"
|
||||
/tmp/praisonai-sandbox-standalone/bin/praisonai-sandbox --help
|
||||
bash "$GITHUB_WORKSPACE/scripts/check_c13_sandbox_imports.sh"
|
||||
python -m pytest src/praisonai-sandbox/tests -q -m "not network"
|
||||
|
||||
- name: Audit hybrid module imports (C8 full stack)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
pip install -q -e src/praisonai-agents -e src/praisonai-code -e src/praisonai-bot -e src/praisonai-train -e src/praisonai-browser -e src/praisonai-mcp -e src/praisonai
|
||||
pip install -q -e src/praisonai-agents -e src/praisonai-code -e src/praisonai-bot -e src/praisonai-train -e src/praisonai-browser -e src/praisonai-mcp -e src/praisonai-sandbox -e src/praisonai
|
||||
python scripts/audit_hybrid_modules.py
|
||||
|
||||
- name: Smoke Test Summary
|
||||
@@ -320,7 +338,6 @@ jobs:
|
||||
--ignore=tests/e2e \
|
||||
--ignore=tests/live \
|
||||
--ignore=tests/unit/jobs \
|
||||
--ignore=tests/unit/sandbox \
|
||||
--ignore=tests/unit/test_langflow_components.py \
|
||||
--cov=praisonai \
|
||||
--cov-report=xml \
|
||||
|
||||
@@ -20,7 +20,7 @@ on:
|
||||
jobs:
|
||||
test-windows:
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 5
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
OPENAI_MODEL_NAME: gpt-4o-mini
|
||||
|
||||
@@ -7,9 +7,9 @@ You are working on the PraisonAI project.
|
||||
- Be concise and helpful in responses
|
||||
- Test implementation thoroughly
|
||||
- Ensure backward compatibility with existing APIs
|
||||
- Follow protocol-driven design across the seven packages: core protocols in `praisonaiagents/`, agentic terminal CLI in `praisonai-code/`, bots/gateway in `praisonai-bot/`, LLM fine-tuning + agent training in `praisonai-train/`, browser automation in `praisonai-browser/`, MCP server host in `praisonai-mcp/`, integrations/serve/dashboard in the `praisonai/` wrapper
|
||||
- Follow protocol-driven design across the eight packages: core protocols in `praisonaiagents/`, agentic terminal CLI in `praisonai-code/`, bots/gateway in `praisonai-bot/`, LLM fine-tuning + agent training in `praisonai-train/`, browser automation in `praisonai-browser/`, MCP server host in `praisonai-mcp/`, sandbox backends in `praisonai-sandbox/`, integrations/serve/dashboard in the `praisonai/` wrapper
|
||||
- Preserve old `praisonai.*` import paths via shims when moving code between packages (see §2.3 in `src/praisonai-agents/AGENTS.md`; shim helpers in `src/praisonai/praisonai/cli/_shim.py`)
|
||||
- Package boundaries and dependency rules: `ARCHITECTURE.md` §2 (Tier 2 packages must never PyPI-depend on the wrapper; cross-tier access goes through lazy `_*_bridge` modules)
|
||||
- Boundary manifests: `src/praisonai/tests/PRAISONAI_BOT_MANIFEST.md` (C9), `src/praisonai/tests/PRAISONAI_TRAIN_MANIFEST.md` (C10), `src/praisonai/tests/PRAISONAI_BROWSER_MANIFEST.md` (C11), `src/praisonai/tests/PRAISONAI_MCP_MANIFEST.md` (C12)
|
||||
- Boundary manifests: `src/praisonai/tests/PRAISONAI_BOT_MANIFEST.md` (C9), `src/praisonai/tests/PRAISONAI_TRAIN_MANIFEST.md` (C10), `src/praisonai/tests/PRAISONAI_BROWSER_MANIFEST.md` (C11), `src/praisonai/tests/PRAISONAI_MCP_MANIFEST.md` (C12), `src/praisonai/tests/PRAISONAI_SANDBOX_MANIFEST.md` (C13)
|
||||
- When reviewing a PR or an issue, evaluate whether the change addresses a framework concern or a user goal, and design its surface (params, naming, defaults) accordingly
|
||||
- The aim of this package is to stay **lightweight and powerful**. Do a critical review at each stage — when triaging an issue, when planning a fix, and when reviewing/implementing a PR. Reject scope creep for the sake of adding features: if a capability already exists (e.g. via existing Agent params like `instructions`/`backstory`/`tools`/`hooks`/`memory`), prefer it over a new API surface. A change must genuinely strengthen the SDK (simpler, more robust, more user-friendly) — do not add knobs, params, modules, or exports that have no live consumer or that merely duplicate existing behaviour.
|
||||
+15
-10
@@ -1,6 +1,6 @@
|
||||
# PraisonAI Architecture
|
||||
|
||||
> **Last updated:** 2026-07-15 (C12 — seven-package tiered model)
|
||||
> **Last updated:** 2026-07-17 (C13 — eight-package tiered model)
|
||||
>
|
||||
> Strategic architecture document for PraisonAI — a multi-agent AI framework.
|
||||
> Covers **Python tiered package model (C7.1 + C9 + C10)**, system design, runtime
|
||||
@@ -11,7 +11,7 @@
|
||||
## Table of Contents
|
||||
|
||||
1. [Executive Summary](#1-executive-summary)
|
||||
2. [Python Tiered Package Model (C7.1 + C9 + C10 + C11 + C12)](#2-python-tiered-package-model-c71--c9--c10--c11--c12)
|
||||
2. [Python Tiered Package Model (C7.1 + C9 + C10 + C11 + C12 + C13)](#2-python-tiered-package-model-c71--c9--c10--c11--c12--c13)
|
||||
3. [System Overview](#3-system-overview)
|
||||
4. [Layered Architecture](#4-layered-architecture)
|
||||
5. [Core Data Contracts](#5-core-data-contracts)
|
||||
@@ -51,16 +51,17 @@ Orchestration + Observability** core to unlock adoption and trust:
|
||||
|
||||
---
|
||||
|
||||
## 2. Python Tiered Package Model (C7.1 + C9 + C10 + C11 + C12)
|
||||
## 2. Python Tiered Package Model (C7.1 + C9 + C10 + C11 + C12 + C13)
|
||||
|
||||
**Release:** v4.6.110+ · `praisonaiagents` · `praisonai-code` · `praisonai-bot` · `praisonai-train` · `praisonai-browser` · `praisonai-mcp` · `praisonai`
|
||||
**Release:** v4.6.110+ · `praisonaiagents` · `praisonai-code` · `praisonai-bot` · `praisonai-train` · `praisonai-browser` · `praisonai-mcp` · `praisonai-sandbox` · `praisonai`
|
||||
|
||||
The Python monorepo publishes seven packages in three tiers with strict dependency
|
||||
The Python monorepo publishes eight packages in three tiers with strict dependency
|
||||
direction. C7 delivered a standalone agentic hot path; C7.1 formalised
|
||||
code/wrapper ownership; C9 extracted bots, gateway, and channel CLI into
|
||||
`praisonai-bot`; C10 extracted LLM fine-tuning and agent training into
|
||||
`praisonai-train`; C11 extracted browser automation into `praisonai-browser`;
|
||||
C12 extracted the heavy MCP host into `praisonai-mcp`.
|
||||
C13 extracted sandbox backends into `praisonai-sandbox`.
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
@@ -68,12 +69,13 @@ flowchart TB
|
||||
Agents["praisonaiagents<br/>Agent, tools, memory, hooks, protocols"]
|
||||
end
|
||||
|
||||
subgraph tier2 [Tier 2 — Terminal + Bot + Train + Browser + MCP]
|
||||
subgraph tier2 [Tier 2 — Terminal + Bot + Train + Browser + MCP + Sandbox]
|
||||
Code["praisonai-code<br/>run, chat, code, Typer, runtime, LLM"]
|
||||
Bot["praisonai-bot<br/>bots, gateway, channel CLI, OS daemon"]
|
||||
Train["praisonai-train<br/>LLM fine-tuning, agent training"]
|
||||
Browser["praisonai-browser<br/>extension bridge, CDP, Playwright"]
|
||||
MCP["praisonai-mcp<br/>MCP server host, auth, capability adapters"]
|
||||
Sandbox["praisonai-sandbox<br/>Docker, E2B, Modal, Sandlock backends"]
|
||||
end
|
||||
|
||||
subgraph tier3 [Tier 3 — Wrapper]
|
||||
@@ -85,32 +87,35 @@ flowchart TB
|
||||
Agents --> Train
|
||||
Agents --> Browser
|
||||
Agents --> MCP
|
||||
Agents --> Sandbox
|
||||
Agents --> Wrapper
|
||||
Code -.->|"lazy _bot_bridge"| Bot
|
||||
Code -.->|"lazy _train_bridge"| Train
|
||||
Code -.->|"lazy _browser_bridge"| Browser
|
||||
Code -.->|"lazy _mcp_bridge"| MCP
|
||||
Code -.->|"lazy _sandbox_bridge"| Sandbox
|
||||
Code -.->|"lazy _wrapper_bridge"| Wrapper
|
||||
Bot -.->|"lazy _code_bridge / _wrapper_bridge"| Code
|
||||
Bot -.->|"lazy _wrapper_bridge"| Wrapper
|
||||
Train -.->|"lazy _code_bridge / _wrapper_bridge"| Code
|
||||
```
|
||||
|
||||
**PyPI publish order:** `praisonaiagents` → `praisonai-code` + `praisonai-bot` + `praisonai-train` + `praisonai-browser` + `praisonai-mcp` → `praisonai`
|
||||
**PyPI publish order:** `praisonaiagents` → `praisonai-code` + `praisonai-bot` + `praisonai-train` + `praisonai-browser` + `praisonai-mcp` + `praisonai-sandbox` → `praisonai`
|
||||
|
||||
**Backward compatibility:** `praisonai.bots`, `praisonai.gateway`, `praisonai.train`,
|
||||
`praisonai.browser`, `praisonai.mcp_server`, and related CLI paths remain as
|
||||
`praisonai.browser`, `praisonai.mcp_server`, `praisonai.sandbox`, and related CLI paths remain as
|
||||
`alias_package` shims to `praisonai_bot.*` / `praisonai_train.*` /
|
||||
`praisonai_browser.*` / `praisonai_mcp.*`.
|
||||
`praisonai_browser.*` / `praisonai_mcp.*` / `praisonai_sandbox.*`.
|
||||
|
||||
| Tier | Package | Owns | Must not depend on |
|
||||
|------|---------|------|-------------------|
|
||||
| 1 | `src/praisonai-agents/` | Agent, tools, memory, hooks, `frameworks/` protocols | `praisonai`, `praisonai-code`, `praisonai-bot`, `praisonai-train`, `praisonai-browser`, `praisonai-mcp` |
|
||||
| 1 | `src/praisonai-agents/` | Agent, tools, memory, hooks, `frameworks/` protocols, sandbox protocols | `praisonai`, `praisonai-code`, `praisonai-bot`, `praisonai-train`, `praisonai-browser`, `praisonai-mcp`, `praisonai-sandbox` |
|
||||
| 2a | `src/praisonai-code/` | `run`/`chat`/`code`, Typer, runtime, LLM, tool resolution | **`praisonai` as a PyPI dependency** (optional lazy imports via `_wrapper_bridge` only) |
|
||||
| 2b | `src/praisonai-bot/` | Bots, gateway, channel CLI, OS daemon, gateway scheduler tick | **`praisonai` as a PyPI dependency** (optional lazy `_wrapper_bridge` for jobs/UI) |
|
||||
| 2c | `src/praisonai-train/` | LLM fine-tuning (Unsloth), agent training, `train` CLI, conda env setup | **`praisonai` as a PyPI dependency** (lazy `_code_bridge` for legacy dispatch) |
|
||||
| 2d | `src/praisonai-browser/` | Extension bridge, CDP/hybrid automation, `browser` CLI | **`praisonai` as a PyPI dependency** (none required; depends on `praisonaiagents` only) |
|
||||
| 2e | `src/praisonai-mcp/` | MCP server host, auth, transports, `mcp` CLI | **`praisonai` as a PyPI dependency** (lazy `_wrapper_bridge` for full capability registry) |
|
||||
| 2f | `src/praisonai-sandbox/` | Sandbox backends (Docker, E2B, Modal, Sandlock, SSH), `SandboxRegistry` | **`praisonai` as a PyPI dependency** (lazy `_code_bridge` for `PluginRegistry`) |
|
||||
| 3 | `src/praisonai/` | `framework_adapters/`, serve, dashboard, async jobs API | — |
|
||||
|
||||
**Config kernel:** Phase 0 `praisonai/common/` was skipped; shared config lives in
|
||||
|
||||
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=4.6.150" \
|
||||
"praisonai>=4.6.155" \
|
||||
"praisonai[chat]" \
|
||||
"embedchain[github,youtube]"
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=4.6.150" \
|
||||
"praisonai>=4.6.155" \
|
||||
"praisonai[ui]" \
|
||||
"praisonai[chat]" \
|
||||
"praisonai[realtime]" \
|
||||
|
||||
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=4.6.150" \
|
||||
"praisonai>=4.6.155" \
|
||||
"praisonai[ui]" \
|
||||
"praisonai[crewai]"
|
||||
|
||||
|
||||
Whitespace-only changes.
@@ -19,7 +19,9 @@ def run_doctor_ci():
|
||||
[sys.executable, "-m", "praisonai", "doctor", "ci"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=120,
|
||||
)
|
||||
|
||||
# Parse JSON output
|
||||
|
||||
@@ -149,21 +149,32 @@ def main():
|
||||
|
||||
print(f"\nOriginal document length: {len(content)} characters")
|
||||
|
||||
chunks = [{"text": content, "metadata": {"source": doc_path}}]
|
||||
|
||||
# Estimate original tokens (~4 chars/token) so each ratio maps to a
|
||||
# distinct token budget — compress() truncates to target_tokens.
|
||||
original_tokens = max(1, len(content) // 4)
|
||||
|
||||
# Compress with different ratios
|
||||
for ratio in [0.3, 0.5, 0.7]:
|
||||
compressor = ContextCompressor(
|
||||
max_tokens=2000,
|
||||
target_ratio=ratio,
|
||||
target_tokens = max(1, int(original_tokens * ratio))
|
||||
compressor = ContextCompressor(verbose=False)
|
||||
result = compressor.compress(
|
||||
chunks,
|
||||
query="security verification code",
|
||||
target_tokens=target_tokens,
|
||||
compression_ratio=ratio,
|
||||
)
|
||||
result = compressor.compress([content], query="security verification code")
|
||||
|
||||
print(f"\nCompression ratio {ratio}:")
|
||||
print(f" Original tokens: {result.original_tokens}")
|
||||
print(f" Compressed tokens: {result.compressed_tokens}")
|
||||
print(f" Actual ratio: {result.compressed_tokens / max(result.original_tokens, 1):.2f}")
|
||||
print(f" Actual ratio: {result.compression_ratio:.2f}")
|
||||
|
||||
# Check if secret code is preserved
|
||||
compressed_text = " ".join(result.chunks) if result.chunks else ""
|
||||
compressed_text = " ".join(
|
||||
c.get("text", "") for c in result.chunks
|
||||
) if result.chunks else ""
|
||||
if SECRET_CODE in compressed_text:
|
||||
print(f" ✅ Secret code preserved in compressed output")
|
||||
else:
|
||||
|
||||
@@ -65,7 +65,6 @@ def main():
|
||||
name="Policy_Expert",
|
||||
instructions="You are a policy expert.",
|
||||
knowledge=[temp_dir],
|
||||
agent_id="policy_agent_v1", # Knowledge scoped to this agent
|
||||
)
|
||||
|
||||
response = shared_agent.chat("What are the expense report rules?")
|
||||
@@ -81,7 +80,6 @@ def main():
|
||||
instructions="You are a personal HR assistant.",
|
||||
knowledge=[temp_dir],
|
||||
memory={"user_id": "user_bob"},
|
||||
agent_id="hr_bot_v2",
|
||||
)
|
||||
|
||||
response = combined_agent.chat("What training is required?")
|
||||
|
||||
@@ -13,7 +13,8 @@ Expected Output:
|
||||
Agent responds with state persisted to Redis
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent, db
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.db import db
|
||||
|
||||
print("=== Redis State Store (Agent-First) ===")
|
||||
|
||||
|
||||
@@ -61,7 +61,7 @@ def demonstrate_collision_problem():
|
||||
_reset_default_registry()
|
||||
|
||||
# Agent 1 registers "custom" provider
|
||||
register_llm_provider("custom", Agent1Provider)
|
||||
register_llm_provider("custom", Agent1Provider, override=True)
|
||||
print("Agent 1 registered 'custom' provider")
|
||||
|
||||
# Agent 2 tries to register same name - ERROR!
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
# praisonai: skip=true
|
||||
# Uses removed praisonaiagents.tools exports; pending example refresh
|
||||
from praisonaiagents import Agent, Tools
|
||||
from praisonaiagents.tools import read_csv, read_excel, write_csv, write_excel, filter_data, get_summary, group_by, pivot_table
|
||||
import os
|
||||
@@ -7,4 +9,4 @@ agent.start(f"""
|
||||
Read the data from the csv file {os.path.join(os.path.dirname(__file__), "tesla-stock-price.csv")}
|
||||
Analyse the data and give me the insights
|
||||
read_csv to read the file
|
||||
""")
|
||||
""")
|
||||
@@ -1,5 +1,7 @@
|
||||
# praisonai: skip=true
|
||||
# Uses removed praisonaiagents.tools exports; pending example refresh
|
||||
from praisonaiagents import Agent, Tools
|
||||
from praisonaiagents.tools import get_stock_price, get_stock_info, get_historical_data
|
||||
|
||||
agent = Agent(instructions="You are a Research Agent", tools=[get_stock_price, get_stock_info, get_historical_data])
|
||||
agent.start("Understand current stock price and historical data of Apple and Google. Tell me if I can invest in them")
|
||||
agent.start("Understand current stock price and historical data of Apple and Google. Tell me if I can invest in them")
|
||||
@@ -135,7 +135,7 @@ def example_auto_agents_multi_provider():
|
||||
from praisonaiagents.agents import AutoAgents
|
||||
|
||||
# Create AutoAgents that will automatically assign appropriate models
|
||||
auto_agents = AutoAgentTeam(
|
||||
auto_agents = AutoAgents(
|
||||
instructions="Create a market research report on electric vehicles. Include data analysis, competitor analysis, and future projections.",
|
||||
max_agents=3,
|
||||
llm="gpt-4o-mini", # Default model for agent generation
|
||||
@@ -144,6 +144,7 @@ def example_auto_agents_multi_provider():
|
||||
|
||||
# After agents are created, upgrade them to multi-model agents
|
||||
multi_model_agents = []
|
||||
wrapper_by_original = {}
|
||||
for agent in auto_agents.agents:
|
||||
# Convert regular agents to multi-model agents
|
||||
multi_agent = RouterAgent(
|
||||
@@ -155,9 +156,18 @@ def example_auto_agents_multi_provider():
|
||||
models=["gpt-4o-mini", "gemini/gemini-1.5-flash", "claude-3-haiku-20240307", "gpt-4o-mini"],
|
||||
routing_strategy="auto")
|
||||
multi_model_agents.append(multi_agent)
|
||||
wrapper_by_original[id(agent)] = multi_agent
|
||||
|
||||
# Update the agents in the AutoAgents instance
|
||||
auto_agents.agents = multi_model_agents
|
||||
|
||||
# Rebind each task's agent to its wrapper — AutoAgents stores a direct agent
|
||||
# reference on every Task and execution reads task.agent, so replacing only
|
||||
# auto_agents.agents would leave the workflow bound to the original agents.
|
||||
for task in getattr(auto_agents, "tasks", []) or []:
|
||||
wrapper = wrapper_by_original.get(id(task.agent))
|
||||
if wrapper is not None:
|
||||
task.agent = wrapper
|
||||
|
||||
# Run the auto-generated workflow
|
||||
results = auto_agents.start()
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
# praisonai: skip=true
|
||||
# Uses removed praisonaiagents.tools exports; pending example refresh
|
||||
from praisonaiagents import Agent, Task, AgentTeam
|
||||
from praisonaiagents.tools import wiki_search, wiki_summary, wiki_page, wiki_random, wiki_language
|
||||
|
||||
@@ -13,4 +15,4 @@ agent.start(
|
||||
"First search the history of AI"
|
||||
"Read the page of the history of AI"
|
||||
"Get the summary of the page"
|
||||
)
|
||||
)
|
||||
@@ -1,3 +1,5 @@
|
||||
# praisonai: skip=true
|
||||
# Requires local audio.mp3 fixture
|
||||
from praisonaiagents import AudioAgent
|
||||
|
||||
agent = AudioAgent(llm="groq/whisper-large-v3") # 10x faster
|
||||
@@ -5,3 +7,4 @@ text = agent.transcribe("audio.mp3", language="en")
|
||||
print(text)
|
||||
|
||||
# Models: openai/whisper-1, groq/whisper-large-v3, deepgram/nova-2
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
# praisonai: skip=true
|
||||
# Requires local audio.mp3 fixture
|
||||
# Speech-to-Text with Groq (fastest)
|
||||
# Requires: export GROQ_API_KEY=your-key
|
||||
|
||||
@@ -6,3 +8,4 @@ from praisonaiagents import AudioAgent
|
||||
agent = AudioAgent(llm="groq/whisper-large-v3")
|
||||
text = agent.listen("audio.mp3") # Replace with your audio file
|
||||
print(text)
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# praisonai: skip=true
|
||||
# Requires local audio.mp3 fixture
|
||||
from praisonaiagents import AudioAgent
|
||||
|
||||
agent = AudioAgent(llm="deepgram/nova-2")
|
||||
text = agent.listen("audio.mp3")
|
||||
print(text)
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# praisonai: skip=true
|
||||
# Requires local audio.mp3 fixture
|
||||
from praisonaiagents import AudioAgent
|
||||
|
||||
agent = AudioAgent(llm="groq/whisper-large-v3")
|
||||
text = agent.listen("audio.mp3")
|
||||
print(text)
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# praisonai: skip=true
|
||||
# Requires local audio.mp3 fixture
|
||||
from praisonaiagents import AudioAgent
|
||||
|
||||
agent = AudioAgent(llm="openai/whisper-1")
|
||||
text = agent.listen("audio.mp3")
|
||||
print(text)
|
||||
|
||||
@@ -18,7 +18,7 @@ from praisonaiagents import AutoAgents
|
||||
from praisonaiagents.tools import duckduckgo
|
||||
|
||||
# Basic usage - AutoAgents analyzes complexity and creates optimal agents
|
||||
agents = AutoAgentTeam(
|
||||
agents = AutoAgents(
|
||||
instructions="Search for information about AI Agents",
|
||||
tools=[duckduckgo],
|
||||
process="sequential", # or "hierarchical"
|
||||
|
||||
@@ -102,7 +102,14 @@ def main():
|
||||
# Print results
|
||||
print("\nAutonomous Agent Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
task_name = result.description
|
||||
print(f"\nTask: {task_name}")
|
||||
|
||||
@@ -64,6 +64,13 @@ results = workflow.start()
|
||||
|
||||
# Print results
|
||||
print("\nEvaluator-Optimizer Results:")
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"Task {task_id}: {result.raw}")
|
||||
@@ -1,4 +1,4 @@
|
||||
from praisonaiagents.agents.agents import Agent, Task, Agents
|
||||
from praisonaiagents.agents.agents import Agent, Task, Agents, AgentTeam
|
||||
from praisonaiagents.tools import duckduckgo
|
||||
|
||||
# Test facts
|
||||
|
||||
@@ -104,6 +104,13 @@ results = workflow.start()
|
||||
|
||||
# Print results
|
||||
print("\nOrchestrator-Workers Results:")
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"Task {task_id}: {result.raw}")
|
||||
@@ -86,7 +86,14 @@ async def main():
|
||||
|
||||
# Print results
|
||||
print("\nParallel Processing Results:")
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"Task {task_id}: {result.raw}")
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from praisonaiagents.agent import Agent
|
||||
from praisonaiagents.task import Task
|
||||
from praisonaiagents.agents import Agents
|
||||
from praisonaiagents.agents import Agents, AgentTeam
|
||||
from typing import List, Dict
|
||||
import time
|
||||
|
||||
@@ -76,6 +76,13 @@ results = workflow.start(return_dict=True)
|
||||
|
||||
# Print results
|
||||
print("\nWorkflow Results:")
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"Task {task_id}: {result.raw}")
|
||||
@@ -132,6 +132,7 @@ if __name__ == "__main__":
|
||||
demonstrate_result_handling()
|
||||
|
||||
print("To run code in sandbox via CLI:")
|
||||
print(" praisonai sandbox run \"print('Hello, World!')\"")
|
||||
print(" praisonai sandbox run --file script.py")
|
||||
print(" praisonai sandbox run --code \"print('Hello, World!')\"")
|
||||
print(" praisonai sandbox run --file script.py --type docker")
|
||||
print(" praisonai sandbox shell")
|
||||
print(" praisonai sandbox backends")
|
||||
@@ -121,7 +121,14 @@ def main():
|
||||
# Print results
|
||||
print("\nAdaptive Learning Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -204,7 +204,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nClimate Impact Analysis Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -97,7 +97,14 @@ def main():
|
||||
# Print results
|
||||
print("\nCode Review Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -222,7 +222,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nCryptography Validation Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -128,7 +128,14 @@ def main():
|
||||
# Print results
|
||||
print("\nCustomer Service Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -208,7 +208,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nMarket Making Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -401,7 +401,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nDisaster Recovery Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -128,7 +128,14 @@ def main():
|
||||
# Print results
|
||||
print("\nEmergency Response Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -125,7 +125,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nFraud Detection Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -153,7 +153,14 @@ def main():
|
||||
# Print results
|
||||
print("\nDiagnosis Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -206,7 +206,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nProtocol Generation Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -150,7 +150,14 @@ def main():
|
||||
# Print results
|
||||
print("\nContent Generation Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -191,7 +191,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nArchitecture Search Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -169,7 +169,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nMaintenance Planning Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -186,7 +186,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nOptimization Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -181,7 +181,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nResearch Analysis Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -175,7 +175,14 @@ def main():
|
||||
# Print results
|
||||
print("\nOptimization Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -220,7 +220,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nResource Optimization Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -102,7 +102,14 @@ def main():
|
||||
# Print results
|
||||
print("\nRisk Management Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -184,7 +184,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nVulnerability Detection Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -4,7 +4,7 @@ Agentic Loop with CSV Workflow Example
|
||||
Demonstrates iterating over a CSV file with an agent processing each row.
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent, Workflow
|
||||
from praisonaiagents import Agent, AgentFlow, Workflow
|
||||
from praisonaiagents.workflows import loop
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
@@ -5,7 +5,7 @@ Demonstrates the evaluator-optimizer pattern where an agent
|
||||
generates content and another evaluates it, repeating until approved.
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent, Workflow
|
||||
from praisonaiagents import Agent, AgentFlow, Workflow
|
||||
from praisonaiagents.workflows import repeat
|
||||
|
||||
# Create generator agent
|
||||
|
||||
@@ -12,7 +12,7 @@ Prerequisites:
|
||||
--port 6060
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent, Workflow
|
||||
from praisonaiagents import Agent, AgentFlow, Workflow
|
||||
|
||||
ROUTELLM_URL = "http://localhost:6060/v1"
|
||||
|
||||
|
||||
@@ -78,7 +78,7 @@ def main():
|
||||
}''')
|
||||
print("\nPress Ctrl+C to stop")
|
||||
|
||||
server.run(transport="sse", host="0.0.0.0", port=8081)
|
||||
server.run_sse(port=8081)
|
||||
|
||||
|
||||
def run_manual_server():
|
||||
|
||||
@@ -1,18 +1,40 @@
|
||||
# Terminal-Bench 2.0 Integration with PraisonAI
|
||||
# Terminal-Bench 2.1 Integration with PraisonAI
|
||||
|
||||
This directory contains examples for integrating PraisonAI Agents with **Terminal-Bench 2.0** via the **Harbor framework** for AI agent benchmarking.
|
||||
This directory contains adapters for benchmarking PraisonAI on **Terminal-Bench 2.1** via the **Harbor framework**. Everything lives in the examples/benchmark layer + CI — there are **zero changes to the `praisonaiagents` / `praisonai` SDK surface**.
|
||||
|
||||
## Overview
|
||||
|
||||
[Terminal-Bench 2.0](https://tbench.ai) is a Stanford/Laude Institute benchmark that has become the gold standard for evaluating AI coding agents in real terminal environments. The [Harbor framework](https://harborframework.com) is the official evaluation harness that abstracts container lifecycle, parallelization, and cloud providers.
|
||||
[Terminal-Bench 2.1](https://www.tbench.ai/docs/run-terminal-bench-2-1) is the current release of the Laude Institute benchmark for evaluating AI coding agents in real terminal environments. The [Harbor framework](https://github.com/laude-institute/harbor) is the official evaluation harness that abstracts container lifecycle, parallelization, and cloud providers.
|
||||
|
||||
## Setup
|
||||
|
||||
Install Harbor and PraisonAI into **one venv** and run from the repo root with `PYTHONPATH=.` so the adapter import path resolves:
|
||||
|
||||
```bash
|
||||
pip install harbor
|
||||
pip install -e src/praisonai-agents -e src/praisonai
|
||||
export PYTHONPATH=.
|
||||
```
|
||||
|
||||
## Integration Types
|
||||
|
||||
### 0. Code Agent (headline — benchmarks `praisonai code`)
|
||||
- **File**: `praisonai_code_agent.py`
|
||||
- **Purpose**: Installs and drives the terminal-native `praisonai code` assistant headlessly inside the container.
|
||||
- **Run**:
|
||||
```bash
|
||||
PYTHONPATH=. harbor run -d terminal-bench/terminal-bench-2-1 \
|
||||
--agent "examples.terminal_bench.praisonai_code_agent:PraisonAICodeAgent" \
|
||||
-m openai/gpt-4o-mini \
|
||||
--ae OPENAI_API_KEY=$OPENAI_API_KEY \
|
||||
-n 4
|
||||
```
|
||||
|
||||
### 1. External Agent (Direct Agent Class)
|
||||
- **File**: `praisonai_external_agent.py`
|
||||
- **Purpose**: External agent that uses direct `Agent()` class instantiation
|
||||
- **Usage**: Run with `--agent-import-path` flag
|
||||
- **Approach**: Uses `praisonaiagents.Agent` directly with `execute_command` tool
|
||||
- **Usage**: Run with `--agent "module:Class"`
|
||||
- **Approach**: Uses `praisonaiagents.Agent` directly with a bash tool bridged to Harbor's `exec()`
|
||||
|
||||
### 2. Wrapper Agent (CLI-Based)
|
||||
- **File**: `praisonai_wrapper_agent.py`
|
||||
@@ -42,12 +64,12 @@ pip install praisonaiagents[tools]
|
||||
|
||||
```bash
|
||||
# Test with oracle agent first
|
||||
harbor run -d terminal-bench/terminal-bench-2 -a oracle
|
||||
harbor run -d terminal-bench/terminal-bench-2-1 -a oracle
|
||||
|
||||
# Run PraisonAI external agent (uses direct Agent() class)
|
||||
harbor run -d terminal-bench/terminal-bench-2 \
|
||||
--agent-import-path examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent \
|
||||
--model openai/gpt-4o \
|
||||
PYTHONPATH=. harbor run -d terminal-bench/terminal-bench-2-1 \
|
||||
--agent "examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent" \
|
||||
--model openai/gpt-4o-mini \
|
||||
--ae OPENAI_API_KEY=$OPENAI_API_KEY \
|
||||
-n 4
|
||||
```
|
||||
@@ -56,9 +78,9 @@ harbor run -d terminal-bench/terminal-bench-2 \
|
||||
|
||||
```bash
|
||||
# Run PraisonAI wrapper agent (uses `praisonai "TASK"` CLI pattern)
|
||||
harbor run -d terminal-bench/terminal-bench-2 \
|
||||
--agent-import-path examples.terminal_bench.praisonai_wrapper_agent:PraisonAIWrapperAgent \
|
||||
--model openai/gpt-4o \
|
||||
PYTHONPATH=. harbor run -d terminal-bench/terminal-bench-2-1 \
|
||||
--agent "examples.terminal_bench.praisonai_wrapper_agent:PraisonAIWrapperAgent" \
|
||||
--model openai/gpt-4o-mini \
|
||||
--ae OPENAI_API_KEY=$OPENAI_API_KEY \
|
||||
-n 4
|
||||
```
|
||||
@@ -66,9 +88,9 @@ harbor run -d terminal-bench/terminal-bench-2 \
|
||||
### Running on Cloud (Daytona/E2B/Modal)
|
||||
|
||||
```bash
|
||||
harbor run -d terminal-bench/terminal-bench-2 \
|
||||
--agent-import-path examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent \
|
||||
--model openai/gpt-4o \
|
||||
PYTHONPATH=. harbor run -d terminal-bench/terminal-bench-2-1 \
|
||||
--agent "examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent" \
|
||||
--model openai/gpt-4o-mini \
|
||||
--env daytona -n 32 \
|
||||
--ae OPENAI_API_KEY=$OPENAI_API_KEY
|
||||
```
|
||||
@@ -100,6 +122,9 @@ harbor run -d terminal-bench/terminal-bench-2 \
|
||||
## Files
|
||||
|
||||
- `README.md` - This documentation
|
||||
- `praisonai_code_agent.py` - **Headline** adapter benchmarking `praisonai code`
|
||||
- `job_code_smoke.yaml` - Smoke-test job config for the code agent
|
||||
- `RESULTS.md` - Recorded pass-rates
|
||||
- `praisonai_external_agent.py` - External agent (direct Agent class)
|
||||
- `praisonai_wrapper_agent.py` - Wrapper agent (CLI-based approach)
|
||||
- `praisonai_installed_agent.py` - Installed agent implementation
|
||||
@@ -127,7 +152,7 @@ Each task provides:
|
||||
|
||||
## Contributing
|
||||
|
||||
1. Test changes with oracle agent first: `harbor run -d terminal-bench/terminal-bench-2 -a oracle`
|
||||
1. Test changes with oracle agent first: `harbor run -d terminal-bench/terminal-bench-2-1 -a oracle`
|
||||
2. Run real agentic tests to ensure end-to-end functionality
|
||||
3. Follow PraisonAI's AGENTS.md architecture guidelines
|
||||
4. Add both unit tests and integration tests
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
# Terminal-Bench 2.1 Results — PraisonAI
|
||||
|
||||
Pass-rates from the Harbor smoke workflow
|
||||
(`.github/workflows/terminal-bench-smoke.yml`, manual dispatch). The full 2.1
|
||||
dataset (89 tasks) can be scored by dropping the `task_names` filter in
|
||||
`job.yaml`.
|
||||
|
||||
| Date | Agent | Model | Tasks | Pass rate | Harbor artifact |
|
||||
|------|-------|-------|-------|-----------|-----------------|
|
||||
| _pending first run_ | `praisonai code` | `openai/gpt-4o-mini` | smoke (3) | — | — |
|
||||
|
||||
## How scores are produced
|
||||
|
||||
```bash
|
||||
PYTHONPATH=. harbor run -c examples/terminal_bench/job_code_smoke.yaml
|
||||
```
|
||||
|
||||
The workflow uploads the Harbor results directory as a CI artifact and appends
|
||||
the pass-rate row above.
|
||||
Whitespace-only changes.
@@ -1,83 +1,20 @@
|
||||
# Harbor Job Configuration for PraisonAI on Terminal-Bench 2.0
|
||||
#
|
||||
# Usage:
|
||||
# harbor run -c examples/terminal_bench/job.yaml
|
||||
# Harbor Job Configuration — PraisonAI external (SDK-level) agent on Terminal-Bench 2.1
|
||||
#
|
||||
# This configuration runs PraisonAI external agent on Terminal-Bench 2.0 tasks
|
||||
# with 8 concurrent trials using GPT-4o.
|
||||
|
||||
# Dataset: Terminal-Bench 2.0 (89 carefully curated terminal tasks)
|
||||
dataset: terminal-bench/terminal-bench-2
|
||||
|
||||
# Agent configuration
|
||||
agent:
|
||||
# Use import path for external agent
|
||||
import_path: examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent
|
||||
|
||||
# Model configuration
|
||||
model_name: openai/gpt-4o
|
||||
|
||||
# Environment variables (API keys)
|
||||
env:
|
||||
OPENAI_API_KEY: "${OPENAI_API_KEY}"
|
||||
# Optional: Add other API keys if using different models
|
||||
# ANTHROPIC_API_KEY: "${ANTHROPIC_API_KEY}"
|
||||
# GEMINI_API_KEY: "${GEMINI_API_KEY}"
|
||||
|
||||
# Execution configuration
|
||||
n_concurrent: 8 # Run 8 trials in parallel
|
||||
n_attempts: 1 # Single attempt per task (benchmark standard)
|
||||
|
||||
# Optional: Environment configuration for cloud execution
|
||||
# environment:
|
||||
# provider: daytona # Options: local, daytona, e2b, modal, runloop, gke
|
||||
# n_concurrent: 32 # Higher concurrency on cloud
|
||||
|
||||
# Optional: Filter to specific tasks for testing
|
||||
# task_filter:
|
||||
# task_names: ["compile_simple_c", "install_python_package", "create_directory"]
|
||||
|
||||
# Optional: Advanced configuration
|
||||
# timeout_sec: 600 # 10 minute timeout per task
|
||||
# save_logs: true # Save execution logs
|
||||
# save_artifacts: true # Save container artifacts
|
||||
|
||||
---
|
||||
|
||||
# Alternative configuration using wrapper agent (CLI-based approach)
|
||||
# Uses `praisonai "TASK"` pattern instead of direct Agent class
|
||||
# Uncomment to use the wrapper approach:
|
||||
|
||||
# dataset: terminal-bench/terminal-bench-2
|
||||
# Run from the repo root with PYTHONPATH set:
|
||||
# PYTHONPATH=. harbor run -c examples/terminal_bench/job.yaml
|
||||
#
|
||||
# agent:
|
||||
# import_path: examples.terminal_bench.praisonai_wrapper_agent:PraisonAIWrapperAgent
|
||||
# model_name: openai/gpt-4o
|
||||
#
|
||||
# n_concurrent: 8
|
||||
# n_attempts: 1
|
||||
#
|
||||
# env:
|
||||
# OPENAI_API_KEY: "${OPENAI_API_KEY}"
|
||||
# Uses the REAL Harbor JobConfig schema. A single YAML document only —
|
||||
# multiple `---` documents make yaml.safe_load raise ComposerError.
|
||||
|
||||
---
|
||||
datasets:
|
||||
- name: terminal-bench
|
||||
version: "2.1"
|
||||
|
||||
# Alternative configuration using installed agent (requires Harbor integration)
|
||||
# Uncomment this section once PraisonAI is integrated into Harbor's codebase
|
||||
agents:
|
||||
- import_path: examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent
|
||||
model_name: openai/gpt-4o-mini
|
||||
env:
|
||||
OPENAI_API_KEY: "${OPENAI_API_KEY}"
|
||||
|
||||
# dataset: terminal-bench/terminal-bench-2
|
||||
#
|
||||
# agent:
|
||||
# name: praisonai
|
||||
# model_name: openai/gpt-4o
|
||||
# flags:
|
||||
# max_turns: 30
|
||||
# verbose: false
|
||||
# memory: false
|
||||
# auto_approval: true
|
||||
#
|
||||
# n_concurrent: 8
|
||||
# n_attempts: 1
|
||||
#
|
||||
# env:
|
||||
# OPENAI_API_KEY: "${OPENAI_API_KEY}"
|
||||
n_concurrent_trials: 4
|
||||
n_attempts: 1
|
||||
@@ -0,0 +1,28 @@
|
||||
# Harbor Job Configuration — `praisonai code` smoke test on Terminal-Bench 2.1
|
||||
#
|
||||
# Run from the repo root with PYTHONPATH set so the adapter import path resolves:
|
||||
# PYTHONPATH=. harbor run -c examples/terminal_bench/job_code_smoke.yaml
|
||||
#
|
||||
# Schema note: this uses the REAL Harbor JobConfig schema
|
||||
# (datasets/agents/n_concurrent_trials/n_attempts). Unknown keys are silently
|
||||
# ignored by pydantic, so the older dataset:/agent:/n_concurrent: form runs the
|
||||
# default oracle agent on zero tasks — do not use it.
|
||||
|
||||
datasets:
|
||||
- name: terminal-bench
|
||||
version: "2.1"
|
||||
# A small verified subset for a cheap smoke run. Verify against the registry
|
||||
# with `harbor datasets list` before editing.
|
||||
task_names:
|
||||
- hello-world
|
||||
- fix-permissions
|
||||
- csv-to-parquet
|
||||
|
||||
agents:
|
||||
- import_path: examples.terminal_bench.praisonai_code_agent:PraisonAICodeAgent
|
||||
model_name: openai/gpt-4o-mini
|
||||
env:
|
||||
OPENAI_API_KEY: "${OPENAI_API_KEY}"
|
||||
|
||||
n_concurrent_trials: 2
|
||||
n_attempts: 1
|
||||
@@ -1,22 +1,24 @@
|
||||
# Harbor Job Configuration for PraisonAI - 5 Task Test
|
||||
# Run with: harbor run -c examples/terminal_bench/job_test_5.yaml -n 1
|
||||
# Harbor Job Configuration — PraisonAI external agent, small verified subset
|
||||
#
|
||||
# Run from the repo root:
|
||||
# PYTHONPATH=. harbor run -c examples/terminal_bench/job_test_5.yaml
|
||||
#
|
||||
# Real Harbor JobConfig schema. Verify task_names against `harbor datasets list`
|
||||
# before editing.
|
||||
|
||||
dataset: terminal-bench/terminal-bench-2
|
||||
datasets:
|
||||
- name: terminal-bench
|
||||
version: "2.1"
|
||||
task_names:
|
||||
- hello-world
|
||||
- fix-permissions
|
||||
- csv-to-parquet
|
||||
|
||||
agent:
|
||||
import_path: examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent
|
||||
model_name: openai/gpt-4o-mini
|
||||
env:
|
||||
OPENAI_API_KEY: "${OPENAI_API_KEY}"
|
||||
agents:
|
||||
- import_path: examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent
|
||||
model_name: openai/gpt-4o-mini
|
||||
env:
|
||||
OPENAI_API_KEY: "${OPENAI_API_KEY}"
|
||||
|
||||
n_concurrent: 1
|
||||
n_concurrent_trials: 1
|
||||
n_attempts: 1
|
||||
|
||||
# Filter to 5 specific simple tasks
|
||||
task_filter:
|
||||
task_names:
|
||||
- "hello-world"
|
||||
- "fix-permissions"
|
||||
- "create-bucket"
|
||||
- "csv-to-parquet"
|
||||
- "simple-web-scraper"
|
||||
@@ -1,22 +1,24 @@
|
||||
# Harbor Job Configuration for PraisonAI Wrapper - 5 Task Test
|
||||
# Run with: harbor run -c examples/terminal_bench/job_test_5_wrapper.yaml -n 1
|
||||
# Harbor Job Configuration — PraisonAI wrapper (CLI) agent, small verified subset
|
||||
#
|
||||
# Run from the repo root:
|
||||
# PYTHONPATH=. harbor run -c examples/terminal_bench/job_test_5_wrapper.yaml
|
||||
#
|
||||
# Real Harbor JobConfig schema. Verify task_names against `harbor datasets list`
|
||||
# before editing.
|
||||
|
||||
dataset: terminal-bench/terminal-bench-2
|
||||
datasets:
|
||||
- name: terminal-bench
|
||||
version: "2.1"
|
||||
task_names:
|
||||
- hello-world
|
||||
- fix-permissions
|
||||
- csv-to-parquet
|
||||
|
||||
agent:
|
||||
import_path: examples.terminal_bench.praisonai_wrapper_agent:PraisonAIWrapperAgent
|
||||
model_name: openai/gpt-4o-mini
|
||||
env:
|
||||
OPENAI_API_KEY: "${OPENAI_API_KEY}"
|
||||
agents:
|
||||
- import_path: examples.terminal_bench.praisonai_wrapper_agent:PraisonAIWrapperAgent
|
||||
model_name: openai/gpt-4o-mini
|
||||
env:
|
||||
OPENAI_API_KEY: "${OPENAI_API_KEY}"
|
||||
|
||||
n_concurrent: 1
|
||||
n_concurrent_trials: 1
|
||||
n_attempts: 1
|
||||
|
||||
# Filter to 5 specific simple tasks
|
||||
task_filter:
|
||||
task_names:
|
||||
- "hello-world"
|
||||
- "fix-permissions"
|
||||
- "create-bucket"
|
||||
- "csv-to-parquet"
|
||||
- "simple-web-scraper"
|
||||
@@ -13,7 +13,7 @@ The multi-agent approach is useful for Terminal-Bench tasks that benefit from:
|
||||
Usage:
|
||||
harbor run -d terminal-bench/terminal-bench-2 \
|
||||
--agent-import-path examples.terminal_bench.multi_agent_example:MultiAgentPraisonAI \
|
||||
--model openai/gpt-4o
|
||||
--model openai/gpt-4o-mini
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -79,7 +79,7 @@ class MultiAgentPraisonAI(BaseAgent):
|
||||
return "Error: Empty command"
|
||||
|
||||
try:
|
||||
result = await environment.exec(command=command, timeout_sec=30)
|
||||
result = await environment.exec(command=command, timeout_sec=300)
|
||||
output_parts = []
|
||||
if result.stdout:
|
||||
output_parts.append(result.stdout.strip())
|
||||
@@ -101,7 +101,7 @@ class MultiAgentPraisonAI(BaseAgent):
|
||||
"Consider potential issues and edge cases. "
|
||||
"Output your plan as a numbered list of specific actions."
|
||||
),
|
||||
llm=self.model_name or "openai/gpt-4o",
|
||||
llm=self.model_name or "openai/gpt-4o-mini",
|
||||
)
|
||||
|
||||
executor = Agent(
|
||||
@@ -114,7 +114,7 @@ class MultiAgentPraisonAI(BaseAgent):
|
||||
"If a step fails, try alternative approaches."
|
||||
),
|
||||
tools=[bash_tool],
|
||||
llm=self.model_name or "openai/gpt-4o",
|
||||
llm=self.model_name or "openai/gpt-4o-mini",
|
||||
)
|
||||
|
||||
verifier = Agent(
|
||||
@@ -127,7 +127,7 @@ class MultiAgentPraisonAI(BaseAgent):
|
||||
"If issues are found, suggest specific fixes."
|
||||
),
|
||||
tools=[bash_tool],
|
||||
llm=self.model_name or "openai/gpt-4o",
|
||||
llm=self.model_name or "openai/gpt-4o-mini",
|
||||
)
|
||||
|
||||
print(f"🚀 Multi-Agent PraisonAI starting: {instruction[:100]}...")
|
||||
@@ -194,7 +194,7 @@ class MultiAgentPraisonAI(BaseAgent):
|
||||
"framework": "praisonai-multi",
|
||||
"agent_type": "multi-agent-team",
|
||||
"agents": [agent.name for agent in agents],
|
||||
"model": self.model_name or "openai/gpt-4o",
|
||||
"model": self.model_name or "openai/gpt-4o-mini",
|
||||
"phases": ["planning", "execution", "verification"],
|
||||
"tools_used": ["bash_tool"],
|
||||
"result_summary": str(result.get("verification", ""))[:200],
|
||||
@@ -238,7 +238,7 @@ class AgentTeamPraisonAI(BaseAgent):
|
||||
try:
|
||||
# Create bash tool
|
||||
async def bash_tool(command: str) -> str:
|
||||
result = await environment.exec(command=command, timeout_sec=30)
|
||||
result = await environment.exec(command=command, timeout_sec=300)
|
||||
output_parts = []
|
||||
if result.stdout:
|
||||
output_parts.append(result.stdout.strip())
|
||||
@@ -252,14 +252,14 @@ class AgentTeamPraisonAI(BaseAgent):
|
||||
planner = Agent(
|
||||
name="planner",
|
||||
instructions="Create detailed execution plans for terminal tasks",
|
||||
llm=self.model_name or "openai/gpt-4o"
|
||||
llm=self.model_name or "openai/gpt-4o-mini"
|
||||
)
|
||||
|
||||
executor = Agent(
|
||||
name="executor",
|
||||
instructions="Execute terminal commands based on plans",
|
||||
tools=[bash_tool],
|
||||
llm=self.model_name or "openai/gpt-4o"
|
||||
llm=self.model_name or "openai/gpt-4o-mini"
|
||||
)
|
||||
|
||||
# Create tasks
|
||||
@@ -317,12 +317,12 @@ if __name__ == "__main__":
|
||||
print("# Multi-agent custom workflow")
|
||||
print("harbor run -d terminal-bench/terminal-bench-2 \\")
|
||||
print(" --agent-import-path examples.terminal_bench.multi_agent_example:MultiAgentPraisonAI \\")
|
||||
print(" --model openai/gpt-4o")
|
||||
print(" --model openai/gpt-4o-mini")
|
||||
print()
|
||||
print("# AgentTeam structured workflow")
|
||||
print("harbor run -d terminal-bench/terminal-bench-2 \\")
|
||||
print(" --agent-import-path examples.terminal_bench.multi_agent_example:AgentTeamPraisonAI \\")
|
||||
print(" --model openai/gpt-4o")
|
||||
print(" --model openai/gpt-4o-mini")
|
||||
print()
|
||||
print("Benefits of multi-agent approach:")
|
||||
print("- Task decomposition and planning")
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
"""
|
||||
PraisonAI Code Agent for Terminal-Bench 2.1 (Harbor)
|
||||
|
||||
The headline adapter: benchmarks the terminal-native `praisonai code` assistant
|
||||
by installing it inside the Harbor container and driving it headlessly.
|
||||
|
||||
Usage:
|
||||
harbor run -d terminal-bench/terminal-bench-2-1 \
|
||||
--agent "examples.terminal_bench.praisonai_code_agent:PraisonAICodeAgent" \
|
||||
-m openai/gpt-4o-mini \
|
||||
--ae OPENAI_API_KEY=$OPENAI_API_KEY \
|
||||
-n 4
|
||||
|
||||
Architecture:
|
||||
Harbor Container → `praisonai code "TASK" --dangerously-skip-approval` (headless)
|
||||
|
||||
Notes:
|
||||
- `--dangerously-skip-approval` sets PRAISON_APPROVAL_MODE=auto +
|
||||
PRAISONAI_TOOL_SAFETY=off so the assistant runs fully autonomously in the
|
||||
container (no approval hang in a non-TTY session).
|
||||
- `praisonai code` normally exits 0 and Harbor grades by task verification,
|
||||
so a benchmark miss does not fail the run — but the real exit status is
|
||||
propagated so genuine install/auth/startup failures still surface.
|
||||
- The base `praisonai` package is sufficient; heavy `code` extras are not
|
||||
required (ACP tools degrade gracefully).
|
||||
|
||||
Dependencies:
|
||||
pip install harbor praisonai praisonaiagents
|
||||
"""
|
||||
|
||||
import shlex
|
||||
|
||||
try:
|
||||
from harbor.agents.installed.base import BaseInstalledAgent
|
||||
from harbor.environments.base import BaseEnvironment
|
||||
from harbor.models.agent.context import AgentContext
|
||||
except ImportError as e: # pragma: no cover - only importable when Harbor present
|
||||
raise ImportError(
|
||||
f"Harbor framework not installed: {e}\n"
|
||||
"Install with: pip install harbor"
|
||||
) from e
|
||||
|
||||
|
||||
class PraisonAICodeAgent(BaseInstalledAgent):
|
||||
"""Benchmarks the `praisonai code` terminal assistant inside a Harbor container."""
|
||||
|
||||
@staticmethod
|
||||
def name() -> str:
|
||||
return "praisonai-code"
|
||||
|
||||
def get_version_command(self) -> str:
|
||||
return "praisonai --version"
|
||||
|
||||
async def install(self, environment: BaseEnvironment) -> None:
|
||||
"""Install python + the praisonai CLI inside the container."""
|
||||
# System packages (best-effort; image may already have them).
|
||||
await self.exec_as_root(
|
||||
environment,
|
||||
command="apt-get update && apt-get install -y python3 python3-pip || true",
|
||||
env={"DEBIAN_FRONTEND": "noninteractive"},
|
||||
)
|
||||
|
||||
# PEP 668-safe install on bookworm-based images.
|
||||
version_spec = f"=={self._version}" if getattr(self, "_version", None) else ""
|
||||
await self.exec_as_agent(
|
||||
environment,
|
||||
command=(
|
||||
f"pip install --break-system-packages praisonai{version_spec} "
|
||||
"praisonaiagents"
|
||||
),
|
||||
)
|
||||
|
||||
async def run(
|
||||
self,
|
||||
instruction: str,
|
||||
environment: BaseEnvironment,
|
||||
context: AgentContext,
|
||||
) -> None:
|
||||
"""Drive `praisonai code` headlessly on the instruction."""
|
||||
model = self.model_name or "openai/gpt-4o-mini"
|
||||
|
||||
command = (
|
||||
f"praisonai code {shlex.quote(instruction)} "
|
||||
f"--dangerously-skip-approval "
|
||||
f"--model {shlex.quote(model)} "
|
||||
"> /tmp/praisonai_code.log 2>&1; "
|
||||
# Capture the real exit status so install/auth/startup failures still
|
||||
# surface instead of being silently masked (Harbor otherwise records a
|
||||
# completed attempt for a crashed agent).
|
||||
"status=$?; "
|
||||
# Tee the log into a stable path so populate_context_post_run can read it.
|
||||
"cp /tmp/praisonai_code.log /tmp/praisonai_code_run.log 2>/dev/null || true; "
|
||||
# `praisonai code` normally exits 0 (Harbor grades by task
|
||||
# verification), so a genuine benchmark miss won't fail here — but a
|
||||
# nonzero status means the assistant itself failed to run.
|
||||
"exit $status"
|
||||
)
|
||||
|
||||
# `extra_env` is the --ae channel; merge it into the exec env (never host
|
||||
# os.environ). `env` on BaseInstalledAgent already includes extra_env.
|
||||
await self.exec_as_agent(
|
||||
environment,
|
||||
command=command,
|
||||
env=self.env,
|
||||
)
|
||||
|
||||
def populate_context_post_run(self, context: AgentContext) -> None:
|
||||
"""`praisonai code` exposes no machine-readable metrics headlessly.
|
||||
|
||||
Record identifying metadata; token/cost accounting is not available from
|
||||
the CLI's mixed stdout, so we leave those fields unset rather than guess.
|
||||
"""
|
||||
context.metadata = {
|
||||
"framework": "praisonai",
|
||||
"agent_type": "code-cli",
|
||||
"agent_name": self.name(),
|
||||
"model": self.model_name,
|
||||
"log_path": "/tmp/praisonai_code.log",
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("PraisonAI Code Agent for Terminal-Bench 2.1")
|
||||
print("Usage:")
|
||||
print(" harbor run -d terminal-bench/terminal-bench-2-1 \\")
|
||||
print(' --agent "examples.terminal_bench.praisonai_code_agent:PraisonAICodeAgent" \\')
|
||||
print(" -m openai/gpt-4o-mini --ae OPENAI_API_KEY=$OPENAI_API_KEY -n 4")
|
||||
@@ -7,7 +7,7 @@ BaseEnvironment interface. The agent uses Harbor's exec() as a bash tool.
|
||||
Usage:
|
||||
harbor run -d terminal-bench/terminal-bench-2 \
|
||||
--agent-import-path examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent \
|
||||
--model openai/gpt-4o \
|
||||
--model openai/gpt-4o-mini \
|
||||
--ae OPENAI_API_KEY=$OPENAI_API_KEY
|
||||
|
||||
Architecture:
|
||||
@@ -75,8 +75,9 @@ class PraisonAIExternalAgent(BaseAgent):
|
||||
"""
|
||||
|
||||
# Inject API keys from Harbor's --ae env vars into host os.environ
|
||||
# so litellm can pick them up (--ae only sets them inside Docker, not the host)
|
||||
agent_env = getattr(context, 'env', {}) or {}
|
||||
# so litellm can pick them up. The --ae channel arrives via the
|
||||
# BaseAgent.extra_env property (AgentContext has no `env` field).
|
||||
agent_env = getattr(self, 'extra_env', {}) or {}
|
||||
for key, val in agent_env.items():
|
||||
if key not in os.environ and val:
|
||||
os.environ[key] = val
|
||||
@@ -139,7 +140,7 @@ class PraisonAIExternalAgent(BaseAgent):
|
||||
" Keep calling bash_tool until the test PASSES or you have exhausted all approaches."
|
||||
),
|
||||
tools=[bash_tool],
|
||||
llm=self.model_name or "openai/gpt-4o",
|
||||
llm=self.model_name or "openai/gpt-4o-mini",
|
||||
)
|
||||
|
||||
# Execute the agent with outer loop to handle premature stopping
|
||||
@@ -182,17 +183,14 @@ class PraisonAIExternalAgent(BaseAgent):
|
||||
Harbor tracks: n_input_tokens, n_output_tokens, cost_usd, metadata
|
||||
"""
|
||||
try:
|
||||
# Extract token usage and cost from agent
|
||||
# Extract token usage and cost from agent. `cost_summary` is a
|
||||
# property returning a dict (not callable) on praisonaiagents.Agent.
|
||||
try:
|
||||
summary = agent.cost_summary() if callable(getattr(agent, 'cost_summary', None)) else None
|
||||
summary = getattr(agent, 'cost_summary', None)
|
||||
if isinstance(summary, dict):
|
||||
context.n_input_tokens = summary.get('tokens_in')
|
||||
context.n_output_tokens = summary.get('tokens_out')
|
||||
context.cost_usd = summary.get('cost')
|
||||
else:
|
||||
context.n_input_tokens = getattr(agent, '_total_tokens_in', 0)
|
||||
context.n_output_tokens = getattr(agent, '_total_tokens_out', 0)
|
||||
context.cost_usd = getattr(agent, 'total_cost', None)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -216,7 +214,7 @@ if __name__ == "__main__":
|
||||
print("PraisonAI External Agent for Terminal-Bench 2.0")
|
||||
print("Usage: harbor run -d terminal-bench/terminal-bench-2 \\")
|
||||
print(" --agent-import-path examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent \\")
|
||||
print(" --model openai/gpt-4o")
|
||||
print(" --model openai/gpt-4o-mini")
|
||||
print()
|
||||
print("Dependencies:")
|
||||
print(" pip install harbor praisonaiagents")
|
||||
@@ -9,7 +9,7 @@ Usage:
|
||||
# First, this would need to be integrated into Harbor's codebase as:
|
||||
# src/harbor/agents/installed/praisonai.py
|
||||
|
||||
harbor run -d terminal-bench/terminal-bench-2 -a praisonai --model openai/gpt-4o
|
||||
harbor run -d terminal-bench/terminal-bench-2 -a praisonai --model openai/gpt-4o-mini
|
||||
|
||||
Architecture:
|
||||
Harbor Container → praisonaiagents installed inside → execute_command tool
|
||||
@@ -25,34 +25,9 @@ import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
try:
|
||||
# These imports would work if this file was in Harbor's codebase
|
||||
from harbor.agents.installed.base import BaseInstalledAgent, with_prompt_template, CliFlag
|
||||
from harbor.environments.base import BaseEnvironment
|
||||
from harbor.models.agent.context import AgentContext
|
||||
from harbor.models.trial.paths import EnvironmentPaths
|
||||
except ImportError:
|
||||
# Fallback for development/testing outside Harbor
|
||||
print("Note: This is a reference implementation for Harbor integration")
|
||||
print("To actually use this, it needs to be integrated into Harbor's codebase")
|
||||
|
||||
# Mock classes for development
|
||||
class BaseInstalledAgent:
|
||||
pass
|
||||
|
||||
def with_prompt_template(func):
|
||||
return func
|
||||
|
||||
class CliFlag:
|
||||
def __init__(self, name, cli, type, default):
|
||||
pass
|
||||
|
||||
class BaseEnvironment:
|
||||
async def exec(self, **kwargs):
|
||||
pass
|
||||
|
||||
class AgentContext:
|
||||
pass
|
||||
from harbor.agents.installed.base import BaseInstalledAgent, with_prompt_template, CliFlag
|
||||
from harbor.environments.base import BaseEnvironment
|
||||
from harbor.models.agent.context import AgentContext
|
||||
|
||||
|
||||
class PraisonAIInstalledAgent(BaseInstalledAgent):
|
||||
@@ -66,12 +41,10 @@ class PraisonAIInstalledAgent(BaseInstalledAgent):
|
||||
# ATIF trajectory support - would be implemented once PraisonAI has trajectory format
|
||||
SUPPORTS_ATIF: bool = False
|
||||
|
||||
# Command-line configuration flags
|
||||
# Command-line configuration flags. The real Harbor CliFlag dataclass keys
|
||||
# the value by `kwarg` (not `name`).
|
||||
CLI_FLAGS = [
|
||||
CliFlag("max_turns", cli="--max-turns", type="int", default=30),
|
||||
CliFlag("verbose", cli="--verbose", type="bool", default=False),
|
||||
CliFlag("memory", cli="--memory", type="bool", default=False),
|
||||
CliFlag("auto_approval", cli="--auto-approval", type="bool", default=True),
|
||||
CliFlag(kwarg="max_turns", cli="--max-turns", type="int", default=30),
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
@@ -142,7 +115,7 @@ def main():
|
||||
sys.exit(1)
|
||||
|
||||
instruction = sys.argv[1]
|
||||
model = sys.argv[2] if len(sys.argv) > 2 else "openai/gpt-4o"
|
||||
model = sys.argv[2] if len(sys.argv) > 2 else "openai/gpt-4o-mini"
|
||||
max_turns = int(sys.argv[3]) if len(sys.argv) > 3 else 30
|
||||
try:
|
||||
from praisonaiagents import Agent
|
||||
@@ -176,19 +149,19 @@ def main():
|
||||
"tools_used": ["execute_command"],
|
||||
}
|
||||
|
||||
# Add token usage if available
|
||||
if hasattr(agent, '_usage'):
|
||||
usage = agent._usage
|
||||
if usage:
|
||||
metrics.update({
|
||||
"input_tokens": getattr(usage, 'input_tokens', None),
|
||||
"output_tokens": getattr(usage, 'output_tokens', None),
|
||||
})
|
||||
|
||||
# Add cost if available
|
||||
if hasattr(agent, '_cost'):
|
||||
metrics["cost_usd"] = agent._cost
|
||||
|
||||
# Token usage + cost come from the `cost_summary` property (a dict).
|
||||
try:
|
||||
summary = getattr(agent, "cost_summary", None)
|
||||
if isinstance(summary, dict):
|
||||
metrics["input_tokens"] = summary.get("tokens_in")
|
||||
metrics["output_tokens"] = summary.get("tokens_out")
|
||||
metrics["cost_usd"] = summary.get("cost")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Write metrics to a stable path so the host adapter can read them back.
|
||||
with open("/tmp/praisonai_metrics.json", "w") as fh:
|
||||
json.dump(metrics, fh)
|
||||
print(json.dumps(metrics))
|
||||
|
||||
except Exception as e:
|
||||
@@ -216,7 +189,7 @@ if __name__ == "__main__":
|
||||
This executes the headless runner script with the instruction,
|
||||
similar to how other installed agents work in Harbor.
|
||||
"""
|
||||
model = self.model_name or "openai/gpt-4o"
|
||||
model = self.model_name or "openai/gpt-4o-mini"
|
||||
max_turns = getattr(self, 'max_turns', 30)
|
||||
|
||||
# Build command to run PraisonAI
|
||||
@@ -227,15 +200,18 @@ if __name__ == "__main__":
|
||||
str(max_turns),
|
||||
]
|
||||
|
||||
command = " ".join(cmd_args)
|
||||
# Tee the metrics file back so populate_context_post_run can read it via
|
||||
# the container. Also print it so it lands in Harbor's captured stdout.
|
||||
command = " ".join(cmd_args) + "; cat /tmp/praisonai_metrics.json 2>/dev/null || true"
|
||||
|
||||
try:
|
||||
# Execute the agent
|
||||
await self.exec_as_agent(
|
||||
# Execute the agent and keep the ExecResult stdout for metric parsing.
|
||||
result = await self.exec_as_agent(
|
||||
environment,
|
||||
command=command,
|
||||
env=self._get_environment_vars(),
|
||||
env=self.env,
|
||||
)
|
||||
self._last_stdout = getattr(result, "stdout", "") or ""
|
||||
except Exception as e:
|
||||
# Store error in context for Harbor's reporting
|
||||
context.metadata = {"execution_error": str(e)}
|
||||
@@ -270,37 +246,38 @@ if __name__ == "__main__":
|
||||
JSON output of the headless runner script.
|
||||
"""
|
||||
try:
|
||||
# Parse the last stdout output for JSON metrics
|
||||
# In Harbor's model, the last execution output should contain our JSON
|
||||
last_output = getattr(context, '_last_stdout', None)
|
||||
|
||||
# Parse the JSON metrics captured from the runner's stdout (teed via
|
||||
# `cat /tmp/praisonai_metrics.json` in run()). AgentContext has no
|
||||
# `_last_stdout`; we stash it on the agent instead.
|
||||
last_output = getattr(self, '_last_stdout', None)
|
||||
metrics = None
|
||||
if last_output:
|
||||
try:
|
||||
metrics = json.loads(last_output.strip())
|
||||
|
||||
# Extract metrics with safe defaults
|
||||
context.n_input_tokens = metrics.get('input_tokens')
|
||||
context.n_output_tokens = metrics.get('output_tokens')
|
||||
context.cost_usd = metrics.get('cost_usd')
|
||||
|
||||
# Store additional metadata
|
||||
context.metadata = {
|
||||
"framework": "praisonai",
|
||||
"agent_type": "installed",
|
||||
"agent_name": metrics.get('agent_name', 'terminal-agent'),
|
||||
"model": metrics.get('model'),
|
||||
"tools_used": metrics.get('tools_used', []),
|
||||
"version": self.get_version() if hasattr(self, 'get_version') else None,
|
||||
}
|
||||
|
||||
except (json.JSONDecodeError, ValueError) as e:
|
||||
# If JSON parsing fails, store basic metadata
|
||||
context.metadata = {
|
||||
"framework": "praisonai",
|
||||
"agent_type": "installed",
|
||||
"parse_error": str(e),
|
||||
"raw_output": str(last_output)[:200] if last_output else None,
|
||||
}
|
||||
# The output may contain CLI noise before the JSON line; grab the
|
||||
# last line that parses as a JSON object.
|
||||
for line in reversed(last_output.strip().splitlines()):
|
||||
line = line.strip()
|
||||
if line.startswith("{"):
|
||||
try:
|
||||
metrics = json.loads(line)
|
||||
break
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
continue
|
||||
|
||||
if metrics is not None:
|
||||
# Extract metrics with safe defaults
|
||||
context.n_input_tokens = metrics.get('input_tokens')
|
||||
context.n_output_tokens = metrics.get('output_tokens')
|
||||
context.cost_usd = metrics.get('cost_usd')
|
||||
|
||||
# Store additional metadata
|
||||
context.metadata = {
|
||||
"framework": "praisonai",
|
||||
"agent_type": "installed",
|
||||
"agent_name": metrics.get('agent_name', 'terminal-agent'),
|
||||
"model": metrics.get('model'),
|
||||
"tools_used": metrics.get('tools_used', []),
|
||||
"version": self.get_version() if hasattr(self, 'get_version') else None,
|
||||
}
|
||||
else:
|
||||
# No output to parse
|
||||
context.metadata = {
|
||||
@@ -334,7 +311,7 @@ if __name__ == "__main__":
|
||||
print(" - Add PraisonAI to _AGENTS list and _AGENT_MAP")
|
||||
print()
|
||||
print("3. Run with Harbor:")
|
||||
print(" harbor run -d terminal-bench/terminal-bench-2 -a praisonai --model openai/gpt-4o")
|
||||
print(" harbor run -d terminal-bench/terminal-bench-2 -a praisonai --model openai/gpt-4o-mini")
|
||||
print()
|
||||
print("Dependencies:")
|
||||
print(" pip install harbor praisonaiagents")
|
||||
@@ -8,7 +8,7 @@ that matches the standard `praisonai "TASK"` CLI usage pattern.
|
||||
Usage:
|
||||
harbor run -d terminal-bench/terminal-bench-2 \
|
||||
--agent-import-path examples.terminal_bench.praisonai_wrapper_agent:PraisonAIWrapperAgent \
|
||||
--model openai/gpt-4o \
|
||||
--model openai/gpt-4o-mini \
|
||||
--ae OPENAI_API_KEY=$OPENAI_API_KEY \
|
||||
-n 4
|
||||
|
||||
@@ -95,13 +95,16 @@ class PraisonAIWrapperAgent(BaseAgent):
|
||||
|
||||
Uses the `praisonai "TASK"` CLI pattern inside the Harbor container.
|
||||
"""
|
||||
model = self.model_name or "openai/gpt-4o"
|
||||
model = self.model_name or "openai/gpt-4o-mini"
|
||||
|
||||
# Build the praisonai CLI command
|
||||
# Format: praisonai "TASK" --model MODEL
|
||||
# Build the `praisonai code` headless command. --dangerously-skip-approval
|
||||
# makes the terminal assistant run autonomously (no approval hang in a
|
||||
# non-TTY container). Bare `praisonai "TASK"` would run agents-generator
|
||||
# mode with no shell-tool loop, so we use the code assistant instead.
|
||||
cmd_parts = [
|
||||
"praisonai",
|
||||
"praisonai", "code",
|
||||
shlex.quote(instruction),
|
||||
"--dangerously-skip-approval",
|
||||
"--model", shlex.quote(model),
|
||||
]
|
||||
|
||||
@@ -148,7 +151,7 @@ class PraisonAIWrapperAgent(BaseAgent):
|
||||
# Store result summary
|
||||
context.metadata = {
|
||||
"agent_name": "praisonai-wrapper",
|
||||
"model": self.model_name or "openai/gpt-4o",
|
||||
"model": self.model_name or "openai/gpt-4o-mini",
|
||||
"framework": "praisonai",
|
||||
"wrapper_type": "cli",
|
||||
"instruction_preview": instruction[:200],
|
||||
@@ -204,16 +207,16 @@ class PraisonAIWrapperAgent(BaseAgent):
|
||||
return env_vars
|
||||
|
||||
async def _exec_as_root(self, environment: BaseEnvironment, command: str, env: Optional[Dict] = None) -> Any:
|
||||
"""Execute command as root in the container."""
|
||||
if hasattr(environment, 'exec_as_root'):
|
||||
return await environment.exec_as_root(command=command, env=env or {})
|
||||
else:
|
||||
# Fallback: use sudo
|
||||
return await environment.exec(command=f"sudo {command}", env=env or {})
|
||||
"""Execute command as root in the container.
|
||||
|
||||
async def _exec_as_agent(self, environment: BaseEnvironment, command: str) -> Any:
|
||||
BaseEnvironment has no `exec_as_root`; use `exec(..., user="root")`.
|
||||
`sudo` is typically absent from TB task containers.
|
||||
"""
|
||||
return await environment.exec(command=command, env=env or {}, user="root")
|
||||
|
||||
async def _exec_as_agent(self, environment: BaseEnvironment, command: str, env: Optional[Dict] = None) -> Any:
|
||||
"""Execute command as the agent user in the container."""
|
||||
return await environment.exec(command=command)
|
||||
return await environment.exec(command=command, env=env or {})
|
||||
|
||||
|
||||
# Example usage for testing
|
||||
@@ -227,7 +230,7 @@ if __name__ == "__main__":
|
||||
print("Usage with Harbor:")
|
||||
print(" harbor run -d terminal-bench/terminal-bench-2 \\")
|
||||
print(" --agent-import-path examples.terminal_bench.praisonai_wrapper_agent:PraisonAIWrapperAgent \\")
|
||||
print(" --model openai/gpt-4o")
|
||||
print(" --model openai/gpt-4o-mini")
|
||||
print()
|
||||
print("Dependencies:")
|
||||
print(" pip install harbor praisonai")
|
||||
@@ -1,274 +1,119 @@
|
||||
"""
|
||||
Tests for PraisonAI Terminal-Bench 2.0 Integration
|
||||
Tests for the PraisonAI Terminal-Bench 2.1 (Harbor) integration.
|
||||
|
||||
This module contains both unit tests and integration tests for the
|
||||
PraisonAI Harbor integration.
|
||||
Run from the repo root with PYTHONPATH set so the package import path resolves:
|
||||
PYTHONPATH=. python -m pytest examples/terminal_bench/test_integration.py -v
|
||||
|
||||
Run tests with:
|
||||
python -m pytest examples/terminal_bench/test_integration.py -v
|
||||
|
||||
Requirements:
|
||||
pip install pytest praisonaiagents
|
||||
# Harbor is optional for unit tests, required for integration tests
|
||||
Tests that require Harbor are honestly skipped only when Harbor is not installed
|
||||
(checked via importlib.util.find_spec). pytest-asyncio must be installed for the
|
||||
async tests.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import shlex
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
import asyncio
|
||||
from unittest.mock import Mock, AsyncMock, patch
|
||||
from pathlib import Path
|
||||
|
||||
HARBOR_AVAILABLE = importlib.util.find_spec("harbor") is not None
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not HARBOR_AVAILABLE, reason="Harbor not installed"
|
||||
)
|
||||
|
||||
|
||||
class TestPraisonAICodeAgent:
|
||||
"""The headline `praisonai code` adapter."""
|
||||
|
||||
def test_metadata(self):
|
||||
from examples.terminal_bench.praisonai_code_agent import PraisonAICodeAgent
|
||||
|
||||
agent = PraisonAICodeAgent(model_name="openai/gpt-4o-mini", logs_dir="/tmp")
|
||||
assert agent.name() == "praisonai-code"
|
||||
assert agent.get_version_command() == "praisonai --version"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_code_agent_command_shape(self, tmp_path):
|
||||
"""run() must issue the exact headless `praisonai code` command."""
|
||||
from examples.terminal_bench.praisonai_code_agent import PraisonAICodeAgent
|
||||
|
||||
agent = PraisonAICodeAgent(
|
||||
model_name="openai/gpt-4o-mini", logs_dir=str(tmp_path)
|
||||
)
|
||||
env = Mock()
|
||||
env.exec = AsyncMock(
|
||||
return_value=SimpleNamespace(stdout="", stderr="", return_code=0)
|
||||
)
|
||||
context = SimpleNamespace(metadata=None)
|
||||
|
||||
instruction = "fix the failing test"
|
||||
await agent.run(instruction, env, context)
|
||||
|
||||
env.exec.assert_awaited()
|
||||
command = env.exec.await_args.kwargs.get("command", "")
|
||||
assert command.startswith(f"praisonai code {shlex.quote(instruction)}")
|
||||
assert "--dangerously-skip-approval" in command
|
||||
assert "--model openai/gpt-4o-mini" in command
|
||||
|
||||
|
||||
class TestPraisonAIExternalAgent:
|
||||
"""Test the external agent implementation."""
|
||||
|
||||
def test_agent_metadata(self):
|
||||
"""Test agent name and version reporting."""
|
||||
try:
|
||||
from .praisonai_external_agent import PraisonAIExternalAgent
|
||||
except ImportError:
|
||||
pytest.skip("Harbor not installed - skipping Harbor-specific tests")
|
||||
|
||||
"""The external (SDK-level) adapter."""
|
||||
|
||||
def test_metadata(self):
|
||||
from examples.terminal_bench.praisonai_external_agent import (
|
||||
PraisonAIExternalAgent,
|
||||
)
|
||||
|
||||
agent = PraisonAIExternalAgent()
|
||||
assert agent.name() == "praisonai"
|
||||
|
||||
# Version should be None if praisonaiagents not installed, or actual version
|
||||
version = agent.version()
|
||||
assert version is None or isinstance(version, str)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_setup(self):
|
||||
"""Test agent setup phase."""
|
||||
try:
|
||||
from .praisonai_external_agent import PraisonAIExternalAgent
|
||||
except ImportError:
|
||||
pytest.skip("Harbor not installed")
|
||||
|
||||
agent = PraisonAIExternalAgent()
|
||||
mock_env = Mock()
|
||||
|
||||
# Setup should complete without error (external agent needs no setup)
|
||||
await agent.setup(mock_env)
|
||||
def test_context_population_uses_cost_summary(self):
|
||||
from examples.terminal_bench.praisonai_external_agent import (
|
||||
PraisonAIExternalAgent,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_tool_execution(self):
|
||||
"""Test the bash tool wrapper around Harbor's exec()."""
|
||||
try:
|
||||
from .praisonai_external_agent import PraisonAIExternalAgent
|
||||
except ImportError:
|
||||
pytest.skip("Harbor not installed")
|
||||
|
||||
# Mock Harbor environment
|
||||
mock_env = Mock()
|
||||
mock_result = Mock()
|
||||
mock_result.stdout = "Hello, World!"
|
||||
mock_result.stderr = ""
|
||||
mock_result.return_code = 0
|
||||
mock_env.exec = AsyncMock(return_value=mock_result)
|
||||
|
||||
# Mock agent context
|
||||
mock_context = Mock()
|
||||
mock_context.metadata = {}
|
||||
|
||||
agent = PraisonAIExternalAgent()
|
||||
|
||||
# Mock PraisonAI Agent to avoid LLM calls in tests
|
||||
with patch('praisonaiagents.Agent') as mock_agent_class:
|
||||
mock_agent_instance = Mock()
|
||||
mock_agent_instance.start.return_value = "Task completed successfully"
|
||||
mock_agent_instance.name = "terminal-agent"
|
||||
mock_agent_class.return_value = mock_agent_instance
|
||||
|
||||
# Mock approval backend
|
||||
with patch('praisonaiagents.approval.set_approval_backend'):
|
||||
await agent.run("echo 'Hello, World!'", mock_env, mock_context)
|
||||
|
||||
# Verify agent was created with correct parameters
|
||||
mock_agent_class.assert_called_once()
|
||||
args, kwargs = mock_agent_class.call_args
|
||||
|
||||
assert kwargs['name'] == 'terminal-agent'
|
||||
assert 'tools' in kwargs
|
||||
assert len(kwargs['tools']) == 1 # bash_tool
|
||||
assert kwargs['verbose'] is False
|
||||
assert kwargs['memory'] is False
|
||||
|
||||
def test_context_population(self):
|
||||
"""Test that agent context is properly populated."""
|
||||
try:
|
||||
from .praisonai_external_agent import PraisonAIExternalAgent
|
||||
except ImportError:
|
||||
pytest.skip("Harbor not installed")
|
||||
|
||||
agent_impl = PraisonAIExternalAgent()
|
||||
|
||||
# Mock agent with usage data
|
||||
|
||||
# cost_summary is a dict property on the real Agent, not a callable.
|
||||
mock_agent = Mock()
|
||||
mock_agent.name = "test-agent"
|
||||
mock_agent.llm = "gpt-4o"
|
||||
mock_agent._usage = Mock()
|
||||
mock_agent._usage.input_tokens = 100
|
||||
mock_agent._usage.output_tokens = 50
|
||||
mock_agent._cost = 0.01
|
||||
|
||||
mock_context = Mock()
|
||||
mock_context.metadata = {}
|
||||
|
||||
# Test context population
|
||||
agent_impl._populate_context(mock_agent, mock_context, "Test result")
|
||||
|
||||
assert mock_context.n_input_tokens == 100
|
||||
assert mock_context.n_output_tokens == 50
|
||||
assert mock_context.cost_usd == 0.01
|
||||
assert mock_context.metadata['agent_name'] == 'test-agent'
|
||||
assert mock_context.metadata['model'] == 'gpt-4o'
|
||||
assert mock_context.metadata['framework'] == 'praisonai'
|
||||
mock_agent.llm = "gpt-4o-mini"
|
||||
mock_agent.cost_summary = {"tokens_in": 100, "tokens_out": 50, "cost": 0.01}
|
||||
|
||||
context = SimpleNamespace(
|
||||
n_input_tokens=None, n_output_tokens=None, cost_usd=None, metadata=None
|
||||
)
|
||||
agent_impl._populate_context(mock_agent, context, "done")
|
||||
|
||||
assert context.n_input_tokens == 100
|
||||
assert context.n_output_tokens == 50
|
||||
assert context.cost_usd == 0.01
|
||||
assert context.metadata["framework"] == "praisonai"
|
||||
|
||||
|
||||
class TestPraisonAIInstalledAgent:
|
||||
"""Test the installed agent implementation."""
|
||||
|
||||
def test_agent_configuration(self):
|
||||
"""Test agent CLI flags and configuration."""
|
||||
try:
|
||||
from .praisonai_installed_agent import PraisonAIInstalledAgent
|
||||
except ImportError:
|
||||
pytest.skip("Harbor not installed")
|
||||
|
||||
agent = PraisonAIInstalledAgent()
|
||||
assert agent.name() == "praisonai"
|
||||
assert agent.SUPPORTS_ATIF is False # Until trajectory format is implemented
|
||||
|
||||
# Check CLI flags
|
||||
flag_names = [flag.name if hasattr(flag, 'name') else str(flag) for flag in agent.CLI_FLAGS]
|
||||
expected_flags = ['max_turns', 'verbose', 'memory', 'auto_approval']
|
||||
for expected in expected_flags:
|
||||
assert any(expected in flag_name for flag_name in flag_names)
|
||||
"""The installed adapter."""
|
||||
|
||||
def test_version_command(self):
|
||||
"""Test version detection command."""
|
||||
try:
|
||||
from .praisonai_installed_agent import PraisonAIInstalledAgent
|
||||
except ImportError:
|
||||
pytest.skip("Harbor not installed")
|
||||
|
||||
agent = PraisonAIInstalledAgent()
|
||||
version_cmd = agent.get_version_command()
|
||||
|
||||
assert version_cmd is not None
|
||||
assert "praisonaiagents" in version_cmd
|
||||
assert "__version__" in version_cmd
|
||||
|
||||
def test_runner_script_generation(self):
|
||||
"""Test that the headless runner script is properly generated."""
|
||||
try:
|
||||
from .praisonai_installed_agent import PraisonAIInstalledAgent
|
||||
except ImportError:
|
||||
pytest.skip("Harbor not installed")
|
||||
|
||||
agent = PraisonAIInstalledAgent()
|
||||
script = agent._build_runner_script()
|
||||
|
||||
assert "import praisonaiagents" in script
|
||||
assert "Agent" in script
|
||||
assert "execute_command" in script
|
||||
assert "AutoApproveBackend" in script
|
||||
assert "json.dumps" in script
|
||||
|
||||
|
||||
class TestIntegration:
|
||||
"""Integration tests that require both PraisonAI and Harbor."""
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_praisonai_agent_real(self):
|
||||
"""
|
||||
Real agentic test - agent must call LLM end-to-end.
|
||||
|
||||
This is a MANDATORY test per AGENTS.md §9.4.
|
||||
Agent MUST call agent.start() with real prompt and produce LLM output.
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents import Agent
|
||||
except ImportError:
|
||||
pytest.skip("PraisonAI not installed")
|
||||
|
||||
# Create real agent that will call LLM
|
||||
agent = Agent(
|
||||
name="test-terminal-agent",
|
||||
instructions="You are a helpful terminal assistant"
|
||||
def test_metadata_and_flags(self):
|
||||
from examples.terminal_bench.praisonai_installed_agent import (
|
||||
PraisonAIInstalledAgent,
|
||||
)
|
||||
|
||||
# Real agentic test - agent must call LLM and produce text response
|
||||
result = agent.start("Say hello in one sentence and mention you can help with terminal tasks")
|
||||
|
||||
# Verify we got actual LLM output
|
||||
assert result is not None
|
||||
assert isinstance(result, str)
|
||||
assert len(result) > 0
|
||||
|
||||
# Print output for manual verification
|
||||
print("✅ Real agentic test result:")
|
||||
print(result)
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_tool_real_execution(self):
|
||||
"""Test bash tool with real command execution (if safe)."""
|
||||
# This would test real bash execution in a safe environment
|
||||
# For now, we'll mock it to avoid system changes
|
||||
|
||||
mock_result = Mock()
|
||||
mock_result.stdout = "PraisonAI Terminal Test\n"
|
||||
mock_result.stderr = ""
|
||||
mock_result.return_code = 0
|
||||
|
||||
# Test that our bash tool wrapper works correctly
|
||||
async def mock_exec(command, timeout_sec=30):
|
||||
assert "echo" in command # Ensure we're testing echo command
|
||||
return mock_result
|
||||
|
||||
# This simulates Harbor's BaseEnvironment.exec()
|
||||
result = await mock_exec("echo 'PraisonAI Terminal Test'")
|
||||
|
||||
assert result.stdout.strip() == "PraisonAI Terminal Test"
|
||||
assert result.return_code == 0
|
||||
agent = PraisonAIInstalledAgent(logs_dir="/tmp")
|
||||
assert agent.name() == "praisonai"
|
||||
# CliFlag is keyed by `kwarg` in the real Harbor dataclass.
|
||||
assert all(getattr(f, "kwarg", None) for f in agent.CLI_FLAGS)
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_auto_approval_setup(self):
|
||||
"""Test that auto-approval backend works correctly."""
|
||||
try:
|
||||
from praisonaiagents.approval import set_approval_backend, AutoApproveBackend
|
||||
except ImportError:
|
||||
pytest.skip("PraisonAI approval system not available")
|
||||
|
||||
# Test setting and restoring approval backend
|
||||
original = set_approval_backend(AutoApproveBackend())
|
||||
new_backend = set_approval_backend(original)
|
||||
|
||||
assert isinstance(new_backend, AutoApproveBackend)
|
||||
def test_runner_script_uses_cost_summary(self):
|
||||
from examples.terminal_bench.praisonai_installed_agent import (
|
||||
PraisonAIInstalledAgent,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Allow running tests directly
|
||||
import sys
|
||||
|
||||
print("PraisonAI Terminal-Bench 2.0 Integration Tests")
|
||||
print("=" * 50)
|
||||
|
||||
# Check dependencies
|
||||
try:
|
||||
import praisonaiagents
|
||||
print(f"✅ PraisonAI version: {getattr(praisonaiagents, '__version__', None)}")
|
||||
except ImportError:
|
||||
print("❌ PraisonAI not installed: pip install praisonaiagents")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
import harbor
|
||||
print("✅ Harbor framework available")
|
||||
except ImportError:
|
||||
print("⚠️ Harbor not installed: pip install harbor")
|
||||
print(" (Some tests will be skipped)")
|
||||
|
||||
print()
|
||||
print("Run tests with: python -m pytest examples/terminal_bench/test_integration.py -v")
|
||||
print("Run real agentic test: python -m pytest examples/terminal_bench/test_integration.py::TestIntegration::test_praisonai_agent_real -v -s")
|
||||
agent = PraisonAIInstalledAgent(logs_dir="/tmp")
|
||||
script = agent._build_runner_script()
|
||||
assert "cost_summary" in script
|
||||
assert "_usage" not in script
|
||||
assert "execute_command" in script
|
||||
@@ -13,7 +13,7 @@ Available presets:
|
||||
- json: JSONL output for piping
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent, Workflow
|
||||
from praisonaiagents import Agent, AgentFlow, Workflow
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
Executable
+69
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env bash
|
||||
# C13 import-direction gate for praisonai-sandbox.
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
||||
CODE_ROOT="${C13_CODE_ROOT:-src/praisonai-code/praisonai_code}"
|
||||
AGENTS_ROOT="${C13_AGENTS_ROOT:-src/praisonai-agents/praisonaiagents}"
|
||||
SANDBOX_ROOT="${C13_SANDBOX_ROOT:-src/praisonai-sandbox/praisonai_sandbox}"
|
||||
|
||||
ANY_WRAPPER_RE='(^[[:space:]]*from praisonai([[:space:]]|\.)|^[[:space:]]*import praisonai($|\.))'
|
||||
ANY_CODE_RE='(^from praisonai_code([[:space:]]|\.)|^import praisonai_code($|\.))'
|
||||
ANY_SANDBOX_RE='(^[[:space:]]*from praisonai_sandbox([[:space:]]|\.)|^[[:space:]]*import praisonai_sandbox($|\.))'
|
||||
|
||||
echo "== C13 praisonai_sandbox wrapper import gate (must be zero outside bridges) =="
|
||||
if command -v rg >/dev/null 2>&1; then
|
||||
MATCHES="$(rg -n "$ANY_WRAPPER_RE" "$SANDBOX_ROOT" --glob '*.py' 2>/dev/null | grep -v '_wrapper_bridge.py' | grep -v '_bootstrap.py' | grep -v '_code_bridge.py' || true)"
|
||||
else
|
||||
MATCHES="$(grep -rEn --include='*.py' "$ANY_WRAPPER_RE" "$SANDBOX_ROOT" 2>/dev/null | grep -v '_wrapper_bridge.py' | grep -v '_bootstrap.py' | grep -v '_code_bridge.py' || true)"
|
||||
fi
|
||||
if [ -n "$MATCHES" ]; then
|
||||
echo "$MATCHES"
|
||||
echo "FAIL: praisonai_sandbox imports the praisonai wrapper"
|
||||
exit 1
|
||||
fi
|
||||
echo "sandbox wrapper import gate ok"
|
||||
|
||||
echo "== C13 praisonai_sandbox module-level praisonai_code import gate =="
|
||||
if command -v rg >/dev/null 2>&1; then
|
||||
MATCHES="$(rg -n "$ANY_CODE_RE" "$SANDBOX_ROOT" --glob '*.py' 2>/dev/null | grep -v '_code_bridge.py' | grep -v '_registry.py' || true)"
|
||||
else
|
||||
MATCHES="$(grep -rEn --include='*.py' "$ANY_CODE_RE" "$SANDBOX_ROOT" 2>/dev/null | grep -v '_code_bridge.py' | grep -v '_registry.py' || true)"
|
||||
fi
|
||||
if [ -n "$MATCHES" ]; then
|
||||
echo "$MATCHES"
|
||||
echo "FAIL: module-level praisonai_code import in praisonai_sandbox"
|
||||
exit 1
|
||||
fi
|
||||
echo "sandbox code import gate ok"
|
||||
|
||||
echo "== C13 praisonaiagents hot-path gate (no praisonai_sandbox at module level) =="
|
||||
for f in \
|
||||
"$AGENTS_ROOT/sandbox/manager.py" \
|
||||
"$AGENTS_ROOT/sandbox/protocols.py" \
|
||||
"$AGENTS_ROOT/sandbox/config.py"
|
||||
do
|
||||
if [ -f "$f" ] && head -n 80 "$f" | grep -E "$ANY_SANDBOX_RE" 2>/dev/null; then
|
||||
echo "FAIL: module-level praisonai_sandbox import in agents hot path $f"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "agents hot-path gate ok"
|
||||
|
||||
echo "== C13 praisonai-code hot-path gate (no praisonai_sandbox at module level) =="
|
||||
for f in \
|
||||
"$CODE_ROOT/cli/main.py" \
|
||||
"$CODE_ROOT/cli/app.py" \
|
||||
"$CODE_ROOT/cli/commands/run.py" \
|
||||
"$CODE_ROOT/cli/commands/chat.py" \
|
||||
"$CODE_ROOT/cli/commands/code.py"
|
||||
do
|
||||
if [ -f "$f" ] && head -n 80 "$f" | grep -E "$ANY_SANDBOX_RE" 2>/dev/null; then
|
||||
echo "FAIL: module-level praisonai_sandbox import in code hot path $f"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "code hot-path gate ok"
|
||||
|
||||
echo "C13 import gates passed"
|
||||
@@ -18,71 +18,71 @@ This report compares **Python SDK feature categories** against **Python document
|
||||
| Category | Features | Docs | Lines |
|
||||
|----------|----------|------|-------|
|
||||
| ✅ AGUI | 1 | 2 | 420 |
|
||||
| ✅ Agent | 21 | 44 | 13662 |
|
||||
| ✅ Agent | 21 | 45 | 14056 |
|
||||
| ✅ Agent-to-Agent (A2A) | 1 | 7 | 2965 |
|
||||
| ✅ Agent-to-User (A2U) | 1 | 3 | 585 |
|
||||
| ✅ Approval | 1 | 5 | 2262 |
|
||||
| ✅ Approval | 1 | 5 | 2286 |
|
||||
| ✅ Audio | 2 | 12 | 1000 |
|
||||
| ✅ Auto Generation | 5 | 9 | 2591 |
|
||||
| ✅ Autonomy | 3 | 7 | 2705 |
|
||||
| ✅ Bots | 7 | 28 | 10942 |
|
||||
| ✅ Auto Generation | 5 | 10 | 2868 |
|
||||
| ✅ Autonomy | 3 | 7 | 2707 |
|
||||
| ✅ Bots | 7 | 28 | 11076 |
|
||||
| ✅ Budget | 1 | 1 | 287 |
|
||||
| ✅ CLI | 5 | 116 | 31593 |
|
||||
| ✅ CLI | 5 | 118 | 32713 |
|
||||
| ✅ Chunking | 2 | 2 | 385 |
|
||||
| ✅ Citations | 2 | 1 | 202 |
|
||||
| ✅ Code Execution | 2 | 13 | 3893 |
|
||||
| ✅ Conditions | 1 | 3 | 1175 |
|
||||
| ✅ Configuration | 3 | 7 | 2763 |
|
||||
| ✅ Context Management | 16 | 34 | 13124 |
|
||||
| ✅ Database | 1 | 40 | 6124 |
|
||||
| ✅ Code Execution | 2 | 13 | 3907 |
|
||||
| ✅ Conditions | 1 | 3 | 1179 |
|
||||
| ✅ Configuration | 3 | 7 | 2772 |
|
||||
| ✅ Context Management | 16 | 34 | 13204 |
|
||||
| ✅ Database | 1 | 40 | 6184 |
|
||||
| ✅ Deep Research | 8 | 2 | 587 |
|
||||
| ✅ Display | 6 | 3 | 741 |
|
||||
| ✅ Embeddings | 6 | 23 | 2049 |
|
||||
| ✅ Evaluation | 1 | 7 | 2829 |
|
||||
| ✅ Evaluation | 1 | 7 | 2902 |
|
||||
| ✅ Events | 1 | 2 | 749 |
|
||||
| ✅ Execution | 3 | 4 | 1528 |
|
||||
| ✅ Failover | 2 | 1 | 400 |
|
||||
| ✅ Files | 2 | 6 | 1964 |
|
||||
| ✅ Flow | 1 | 3 | 777 |
|
||||
| ✅ Gateway | 7 | 48 | 16453 |
|
||||
| ✅ Gateway | 7 | 52 | 18697 |
|
||||
| ✅ Guardrails | 4 | 4 | 1673 |
|
||||
| ✅ Handoffs | 11 | 6 | 2454 |
|
||||
| ✅ Hooks | 2 | 9 | 3850 |
|
||||
| ✅ Hooks | 2 | 9 | 3981 |
|
||||
| ✅ Image | 1 | 11 | 1255 |
|
||||
| ✅ Knowledge | 4 | 13 | 4160 |
|
||||
| ✅ LLM | 3 | 14 | 4767 |
|
||||
| ✅ Loops | 4 | 4 | 1075 |
|
||||
| ✅ MCP | 1 | 56 | 12475 |
|
||||
| ✅ Memory | 6 | 17 | 6475 |
|
||||
| ✅ Loops | 4 | 4 | 1202 |
|
||||
| ✅ MCP | 1 | 58 | 13322 |
|
||||
| ✅ Memory | 6 | 17 | 6635 |
|
||||
| ✅ OCR | 2 | 1 | 237 |
|
||||
| ✅ Observability | 2 | 23 | 2771 |
|
||||
| ✅ Observability | 2 | 23 | 2779 |
|
||||
| ✅ Optimizer | 1 | 2 | 792 |
|
||||
| ✅ Output | 3 | 5 | 1158 |
|
||||
| ✅ Parallel Execution | 3 | 2 | 497 |
|
||||
| ✅ Planning | 6 | 6 | 1545 |
|
||||
| ✅ Plugins | 8 | 3 | 1879 |
|
||||
| ✅ Prompts | 2 | 7 | 1452 |
|
||||
| ✅ Providers | 1 | 55 | 7455 |
|
||||
| ✅ Prompts | 2 | 8 | 1726 |
|
||||
| ✅ Providers | 1 | 55 | 7459 |
|
||||
| ✅ Query | 1 | 2 | 736 |
|
||||
| ✅ RAG | 5 | 15 | 2936 |
|
||||
| ✅ Realtime | 2 | 5 | 668 |
|
||||
| ✅ Reflection | 3 | 3 | 680 |
|
||||
| ✅ Retrieval | 2 | 5 | 1044 |
|
||||
| ✅ Routing | 1 | 2 | 430 |
|
||||
| ✅ Sandbox | 5 | 6 | 2559 |
|
||||
| ✅ Security | 1 | 3 | 2863 |
|
||||
| ✅ Sessions | 4 | 13 | 4971 |
|
||||
| ✅ Skills | 6 | 13 | 4893 |
|
||||
| ✅ Sandbox | 5 | 6 | 2700 |
|
||||
| ✅ Security | 1 | 3 | 2903 |
|
||||
| ✅ Sessions | 4 | 13 | 5286 |
|
||||
| ✅ Skills | 6 | 13 | 4946 |
|
||||
| ✅ Tasks | 2 | 6 | 2743 |
|
||||
| ✅ Telemetry | 1 | 2 | 604 |
|
||||
| ✅ Templates | 1 | 8 | 1657 |
|
||||
| ✅ Tools | 12 | 131 | 36741 |
|
||||
| ✅ Tools | 12 | 132 | 37466 |
|
||||
| ✅ Tracing | 3 | 2 | 139 |
|
||||
| ✅ Vector Store | 1 | 12 | 1193 |
|
||||
| ✅ Video | 2 | 6 | 649 |
|
||||
| ✅ Vision | 2 | 1 | 329 |
|
||||
| ✅ Web | 3 | 8 | 2055 |
|
||||
| ✅ Workflows | 5 | 16 | 6435 |
|
||||
| ✅ Workflows | 5 | 16 | 6527 |
|
||||
|
||||
## Documentation Without Features
|
||||
|
||||
|
||||
@@ -515,8 +515,7 @@ _LAZY_IMPORTS = {
|
||||
'ManagerConfig': ('praisonaiagents.context.manager', 'ManagerConfig'),
|
||||
'ContextManager': ('praisonaiagents.context.manager', 'ContextManager'),
|
||||
|
||||
# db module
|
||||
'db': ('praisonaiagents.db', 'db'),
|
||||
# db shortcut (handled by custom_handler to return _LazyDbModule instance)
|
||||
# Note: 'obs' is handled by custom_handler to return _LazyObsModule instance
|
||||
|
||||
# Gateway protocols and config (implementations in praisonai wrapper)
|
||||
@@ -711,9 +710,15 @@ def _custom_handler(name, cache):
|
||||
return mod.obs
|
||||
if name == 'db':
|
||||
import importlib
|
||||
import sys
|
||||
mod = importlib.import_module('.db', 'praisonaiagents')
|
||||
cache['db'] = mod
|
||||
return mod
|
||||
# importlib registers the submodule as praisonaiagents.db in the package
|
||||
# namespace, so plain attribute lookup would bypass __getattr__ and return
|
||||
# the raw module on repeat access. Rebind the package global to the callable
|
||||
# _LazyDbModule proxy so `praisonaiagents.db(...)` keeps working every time.
|
||||
cache['db'] = mod.db # Return the _LazyDbModule instance, not the module
|
||||
setattr(sys.modules['praisonaiagents'], 'db', mod.db)
|
||||
return mod.db
|
||||
if name == 'toolsets':
|
||||
import importlib
|
||||
mod = importlib.import_module('.toolsets', 'praisonaiagents')
|
||||
|
||||
@@ -1273,16 +1273,42 @@ class Agent(GoalLoopMixin, SteeringMixin, SandboxMixin, SkillReviewMixin, Unifie
|
||||
session_id = _history_session_id
|
||||
elif _history_enabled and session_id is None and _history_session_id is None:
|
||||
import hashlib as _hl
|
||||
_agent_hash = _hl.sha256((name or "agent").encode()).hexdigest()[:8]
|
||||
# Backward compat: check if legacy md5-based session exists first
|
||||
_legacy_hash = _hl.md5((name or "agent").encode()).hexdigest()[:8]
|
||||
_legacy_id = f"history_{_legacy_hash}"
|
||||
_new_id = f"history_{_agent_hash}"
|
||||
# Prefer legacy if it exists on disk, else use new SHA-256 ID
|
||||
import os as _os
|
||||
_session_dir = _os.path.join(_os.path.expanduser("~"), ".praisonai", "sessions")
|
||||
if _os.path.exists(_os.path.join(_session_dir, f"{_legacy_id}.json")):
|
||||
session_id = _legacy_id # preserve existing history
|
||||
_name = name or "agent"
|
||||
# Workspace-scoped id so same-named agents in different projects don't
|
||||
# collide (Issue #3154). Opt out with PRAISONAI_GLOBAL_SESSIONS=true for
|
||||
# name-only global continuity.
|
||||
_global_scope = _os.environ.get("PRAISONAI_GLOBAL_SESSIONS", "").lower() in ("1", "true", "yes")
|
||||
if _global_scope:
|
||||
_workspace_id = "global"
|
||||
else:
|
||||
from ..session.workspace import workspace_id as _wid
|
||||
_workspace_id = _wid()
|
||||
_workspace_hash = _hl.sha256(f"{_workspace_id}:{_name}".encode()).hexdigest()[:8]
|
||||
_new_id = f"history_{_workspace_hash}"
|
||||
# Backward-compat ids (resolution order: name-only sha256, then md5)
|
||||
_name_sha_id = f"history_{_hl.sha256(_name.encode()).hexdigest()[:8]}"
|
||||
_name_md5_id = f"history_{_hl.md5(_name.encode()).hexdigest()[:8]}"
|
||||
# Resolve the migration lookup against the SAME directory the session
|
||||
# store uses at runtime, so a custom PRAISONAI_HOME is honoured (the
|
||||
# store reads from get_sessions_dir()). Fall back to the default path
|
||||
# only if that helper is unavailable.
|
||||
try:
|
||||
from ..paths import get_sessions_dir as _get_sessions_dir
|
||||
_session_dir = str(_get_sessions_dir())
|
||||
except Exception:
|
||||
_session_dir = _os.path.join(_os.path.expanduser("~"), ".praisonai", "sessions")
|
||||
# Prefer an existing workspace-scoped session. Legacy (pre-workspace)
|
||||
# name-only files are only adopted in global scope: doing so in a
|
||||
# workspace-scoped run would silently share Project A's history with a
|
||||
# same-named agent in Project B (Issue #3154). Global scope is the
|
||||
# explicit opt-in for that name-only continuity.
|
||||
if _os.path.exists(_os.path.join(_session_dir, f"{_new_id}.json")):
|
||||
session_id = _new_id
|
||||
elif _global_scope and _os.path.exists(_os.path.join(_session_dir, f"{_name_sha_id}.json")):
|
||||
session_id = _name_sha_id # preserve existing history
|
||||
elif _global_scope and _os.path.exists(_os.path.join(_session_dir, f"{_name_md5_id}.json")):
|
||||
session_id = _name_md5_id # preserve existing history
|
||||
else:
|
||||
session_id = _new_id
|
||||
_history_session_id = session_id
|
||||
@@ -1759,6 +1785,7 @@ class Agent(GoalLoopMixin, SteeringMixin, SandboxMixin, SkillReviewMixin, Unifie
|
||||
if toolsets:
|
||||
try:
|
||||
from ..toolsets import resolve_toolsets_for_model
|
||||
from ..tools.resolver import ToolResolutionError
|
||||
# Advertise the model-family's preferred edit primitive first
|
||||
# (e.g. apply_patch for Claude, edit_file for GPT). Unknown /
|
||||
# non-string models fall back to the byte-for-byte default order.
|
||||
@@ -1774,6 +1801,11 @@ class Agent(GoalLoopMixin, SteeringMixin, SandboxMixin, SkillReviewMixin, Unifie
|
||||
toolset_tools = self._resolve_tool_names(unique_tool_names)
|
||||
self.tools.extend(toolset_tools)
|
||||
logging.debug(f"Resolved toolsets {toolsets} to {len(toolset_tools)} tools: {[getattr(t, '__name__', str(t)) for t in toolset_tools]}")
|
||||
except ToolResolutionError:
|
||||
# Preserve the typed error (with .unknown / .suggestions) so
|
||||
# strict-mode callers get the same self-correcting contract as
|
||||
# the direct tools= path instead of a flattened ValueError.
|
||||
raise
|
||||
except (ValueError, KeyError) as e:
|
||||
raise ValueError(
|
||||
f"Agent '{getattr(self, 'display_name', 'unknown')}' failed to resolve toolsets {toolsets}: {e}. "
|
||||
@@ -2085,15 +2117,18 @@ Your Goal: {self.goal}
|
||||
# Store tool retry policy for tool execution with exponential backoff
|
||||
self._tool_retry_policy = _tool_config.retry_policy if _tool_config else None
|
||||
|
||||
# Retry configuration with jittered exponential backoff
|
||||
# Retry configuration with jittered exponential backoff.
|
||||
# Default (retry is None) applies RetryBackoffConfig() so the native
|
||||
# OpenAI-client path retries transient errors by default, matching the
|
||||
# LiteLLM path (max_retries=3). Only retry=False disables retries.
|
||||
if isinstance(retry, RetryBackoffConfig):
|
||||
self._retry_config = retry
|
||||
elif isinstance(retry, dict):
|
||||
self._retry_config = RetryBackoffConfig(**retry)
|
||||
elif retry is True:
|
||||
self._retry_config = RetryBackoffConfig() # Use defaults
|
||||
elif retry is False:
|
||||
self._retry_config = None # Explicitly disabled
|
||||
else:
|
||||
self._retry_config = None # No retry configuration
|
||||
self._retry_config = RetryBackoffConfig() # Use defaults (retry is True or None)
|
||||
|
||||
# Cache for system prompts and formatted tools with eager thread-safe lock
|
||||
# Use OrderedDict for LRU behavior
|
||||
@@ -4702,6 +4737,27 @@ Summary:"""
|
||||
logger.warning(f"Runtime MCP server '{name}' cleanup failed: {e}")
|
||||
servers.clear()
|
||||
|
||||
def _cleanup_circuit_breakers(self) -> None:
|
||||
"""Remove this agent's instance-scoped tool circuit breakers.
|
||||
|
||||
Breakers are keyed on ``id(self)`` in a process-global registry. Since
|
||||
CPython may reuse an object id after this agent is collected, leaving the
|
||||
entries behind could let a future agent inherit a stale OPEN breaker.
|
||||
Removing them on close keeps the registry bounded and correct.
|
||||
"""
|
||||
try:
|
||||
from ..tools.circuit_breaker import _get_global_registry
|
||||
except Exception:
|
||||
return
|
||||
try:
|
||||
registry = _get_global_registry()
|
||||
prefix = f"tool_{id(self)}_"
|
||||
for name in registry.list_services():
|
||||
if name.startswith(prefix):
|
||||
registry.remove(name)
|
||||
except Exception as e:
|
||||
logger.warning(f"Circuit breaker cleanup failed: {e}")
|
||||
|
||||
def _model_supports_web_search(self) -> bool:
|
||||
"""
|
||||
Check if the agent's model supports native web search via LiteLLM.
|
||||
@@ -5507,7 +5563,9 @@ Answer:"""
|
||||
elif isinstance(self.guardrail, str):
|
||||
# Create LLM-based guardrail
|
||||
from ..guardrails import LLMGuardrail
|
||||
llm = getattr(self, 'llm', None) or getattr(self, 'llm_instance', None)
|
||||
# Prefer the configured LLM instance (with api_key/base_url/client
|
||||
# overrides) over the bare model-name string in self.llm.
|
||||
llm = getattr(self, 'llm_instance', None) or getattr(self, 'llm', None)
|
||||
self._guardrail_fn = LLMGuardrail(description=self.guardrail, llm=llm)
|
||||
else:
|
||||
raise ValueError("Agent guardrail must be either a callable or a string description")
|
||||
@@ -6141,19 +6199,26 @@ Answer:"""
|
||||
except Exception as e:
|
||||
logger.warning(f"LLM client cleanup failed: {e}")
|
||||
|
||||
# MCP cleanup
|
||||
# MCP cleanup — shut down MCP clients passed via tools=[MCP(...)]
|
||||
# (mirrors remove_mcp_server()'s best-effort shutdown)
|
||||
try:
|
||||
if hasattr(self, '_mcp_clients') and self._mcp_clients:
|
||||
for client_name, client in self._mcp_clients.items():
|
||||
if hasattr(client, 'close'):
|
||||
client.close()
|
||||
self._mcp_clients.clear()
|
||||
if isinstance(self.tools, list):
|
||||
for t in self.tools:
|
||||
if hasattr(t, 'shutdown'):
|
||||
try:
|
||||
t.shutdown()
|
||||
except Exception as e:
|
||||
logger.warning(f"MCP tool cleanup failed: {e}")
|
||||
except Exception as e:
|
||||
logger.warning(f"MCP cleanup failed: {e}")
|
||||
|
||||
# Runtime-attached MCP servers cleanup (each guarded individually)
|
||||
self._shutdown_runtime_mcp_servers()
|
||||
|
||||
# Circuit breaker cleanup — remove this agent's instance-scoped breakers
|
||||
# so a reused id(self) can't inherit a stale OPEN breaker.
|
||||
self._cleanup_circuit_breakers()
|
||||
|
||||
# Server registry cleanup
|
||||
try:
|
||||
self._cleanup_server_registrations()
|
||||
@@ -6194,18 +6259,27 @@ Answer:"""
|
||||
elif hasattr(self.memory, 'close_connections'):
|
||||
self.memory.close_connections()
|
||||
|
||||
# Close MCP sessions asynchronously if supported
|
||||
if hasattr(self, '_mcp_clients') and self._mcp_clients:
|
||||
for client in self._mcp_clients.values():
|
||||
if hasattr(client, 'aclose'):
|
||||
await client.aclose()
|
||||
elif hasattr(client, 'close'):
|
||||
client.close()
|
||||
self._mcp_clients.clear()
|
||||
# Close MCP clients passed via tools=[MCP(...)]
|
||||
# (mirrors remove_mcp_server()'s best-effort shutdown)
|
||||
if isinstance(self.tools, list):
|
||||
for t in self.tools:
|
||||
if hasattr(t, 'aclose'):
|
||||
try:
|
||||
await t.aclose()
|
||||
except Exception as e:
|
||||
logger.warning(f"MCP tool cleanup failed: {e}")
|
||||
elif hasattr(t, 'shutdown'):
|
||||
try:
|
||||
t.shutdown()
|
||||
except Exception as e:
|
||||
logger.warning(f"MCP tool cleanup failed: {e}")
|
||||
|
||||
# Runtime-attached MCP servers cleanup (each guarded individually)
|
||||
self._shutdown_runtime_mcp_servers()
|
||||
|
||||
# Circuit breaker cleanup — remove this agent's instance-scoped breakers
|
||||
self._cleanup_circuit_breakers()
|
||||
|
||||
# Clean up server registrations and tasks
|
||||
self._cleanup_server_registrations()
|
||||
|
||||
|
||||
@@ -106,7 +106,10 @@ class ChatMixin:
|
||||
|
||||
cached_prompt = self._cache_get(self._system_prompt_cache, cache_key)
|
||||
if cached_prompt is not None:
|
||||
return cached_prompt
|
||||
# Path-scoped glob rules are per-turn (they depend on the files
|
||||
# touched so far) so they are appended after the cache, never
|
||||
# baked into the cached base prompt.
|
||||
return self._append_glob_rules_context(cached_prompt)
|
||||
else:
|
||||
cache_key = None # Don't cache when memory is enabled
|
||||
|
||||
@@ -279,7 +282,87 @@ Your Goal: {self.goal}"""
|
||||
pass # Trust module not available, skip security instructions
|
||||
|
||||
# Note: Caching is done BEFORE session context injection to avoid cross-user leakage
|
||||
return system_prompt
|
||||
# Append per-turn path-scoped (glob) rules last so they are never cached.
|
||||
return self._append_glob_rules_context(system_prompt)
|
||||
|
||||
def _append_glob_rules_context(self, system_prompt):
|
||||
"""Append path-scoped (activation: glob) rules for files touched this run.
|
||||
|
||||
Path-scoped rules activate dynamically: as the agent reads or edits
|
||||
files during a run, any glob rule whose pattern matches a touched path
|
||||
is injected once, deduplicated against the always/manual rules already
|
||||
present in the base prompt. This is per-turn context, so it is added
|
||||
after the (cached) base prompt and never baked into the cache.
|
||||
|
||||
The whole path is gated on ``has_glob_rules()`` so workspaces without
|
||||
any glob rules incur zero cost.
|
||||
"""
|
||||
if not system_prompt:
|
||||
return system_prompt
|
||||
if not (getattr(self, "_rules_enabled", True) and self.rules_manager):
|
||||
return system_prompt
|
||||
manager = self.rules_manager
|
||||
if not getattr(manager, "has_glob_rules", None) or not manager.has_glob_rules():
|
||||
return system_prompt
|
||||
|
||||
file_paths = self._collect_touched_file_paths()
|
||||
if not file_paths:
|
||||
return system_prompt
|
||||
|
||||
# Deduplicate against rules already emitted in the base prompt.
|
||||
already = {r.name for r in manager.get_active_rules()}
|
||||
matched = manager.get_glob_rules_for_paths(list(file_paths), exclude_names=already)
|
||||
if not matched:
|
||||
return system_prompt
|
||||
|
||||
sections = []
|
||||
for rule in matched:
|
||||
text = (rule.content or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
header = f"## {rule.name}: {rule.description}" if rule.description else f"## {rule.name}"
|
||||
sections.append(f"{header}\n{text}")
|
||||
if not sections:
|
||||
return system_prompt
|
||||
|
||||
return (
|
||||
system_prompt
|
||||
+ "\n\n## Path-Scoped Rules (apply to the files being worked on)\n"
|
||||
+ "\n\n".join(sections)
|
||||
)
|
||||
|
||||
def _collect_touched_file_paths(self):
|
||||
"""Extract candidate file paths referenced in the current conversation.
|
||||
|
||||
Scans user prompts and tool results in chat history for path-like
|
||||
tokens (containing a supported extension). This gives dynamic, per-turn
|
||||
glob activation without threading a path argument through every tool
|
||||
call site: as the agent reads/edits files, matching glob rules appear.
|
||||
"""
|
||||
history = getattr(self, "chat_history", None)
|
||||
if not history:
|
||||
return set()
|
||||
|
||||
pattern = getattr(self, "_glob_path_pattern", None)
|
||||
if pattern is None:
|
||||
import re as _re
|
||||
# Match bare or quoted path tokens ending in a file extension, e.g.
|
||||
# foo.py, src/main.py, "tests/test_x.py".
|
||||
pattern = _re.compile(r"[\w./\\-]+\.[A-Za-z0-9]{1,8}")
|
||||
self._glob_path_pattern = pattern
|
||||
|
||||
paths = set()
|
||||
for msg in history:
|
||||
if not isinstance(msg, dict):
|
||||
continue
|
||||
content = msg.get("content")
|
||||
if not isinstance(content, str) or not content:
|
||||
continue
|
||||
for match in pattern.findall(content):
|
||||
normalized = match.strip("'\"").replace("\\", "/")
|
||||
if "." in normalized:
|
||||
paths.add(normalized)
|
||||
return paths
|
||||
|
||||
def _build_response_format(self, schema_model):
|
||||
"""Build response_format dict for native structured output.
|
||||
@@ -644,10 +727,12 @@ Your Goal: {self.goal}"""
|
||||
Returns the possibly-mutated list of tool definitions. Synchronous
|
||||
path; use ``_aapply_before_tool_definitions_hook`` in async contexts.
|
||||
"""
|
||||
if not formatted_tools or not getattr(self, '_hook_runner', None):
|
||||
from ..hooks import HookEvent
|
||||
_runner = getattr(self, '_hook_runner', None)
|
||||
if (not formatted_tools or _runner is None
|
||||
or not _runner.registry.has_hooks(HookEvent.BEFORE_TOOL_DEFINITIONS)):
|
||||
return formatted_tools
|
||||
try:
|
||||
from ..hooks import HookEvent
|
||||
_inp = self._build_before_tool_definitions_input(formatted_tools)
|
||||
_results = self._hook_runner.execute_sync(HookEvent.BEFORE_TOOL_DEFINITIONS, _inp)
|
||||
# Fail closed on a blocking hook/plugin: a POLICY/GUARDRAIL that
|
||||
@@ -676,10 +761,12 @@ Your Goal: {self.goal}"""
|
||||
``execute_sync`` raises inside a running event loop, so the async chat
|
||||
path must await the hook runner directly to actually run the hook.
|
||||
"""
|
||||
if not formatted_tools or not getattr(self, '_hook_runner', None):
|
||||
from ..hooks import HookEvent
|
||||
_runner = getattr(self, '_hook_runner', None)
|
||||
if (not formatted_tools or _runner is None
|
||||
or not _runner.registry.has_hooks(HookEvent.BEFORE_TOOL_DEFINITIONS)):
|
||||
return formatted_tools
|
||||
try:
|
||||
from ..hooks import HookEvent
|
||||
_inp = self._build_before_tool_definitions_input(formatted_tools)
|
||||
_results = await self._hook_runner.execute(HookEvent.BEFORE_TOOL_DEFINITIONS, _inp)
|
||||
# Fail closed on a blocking hook/plugin (see sync variant).
|
||||
@@ -1132,6 +1219,7 @@ Your Goal: {self.goal}"""
|
||||
|
||||
async def _apply_compaction_async(self, messages, compactor, policy):
|
||||
"""Async version of _apply_compaction."""
|
||||
import asyncio
|
||||
from ..compaction.strategy import CompactionStrategy as LegacyStrategy
|
||||
from ..hooks import HookEvent as _HookEvent
|
||||
import logging
|
||||
@@ -1161,7 +1249,18 @@ Your Goal: {self.goal}"""
|
||||
f"[proactive-compaction-async] {self.name}: {result.original_tokens}→{result.compacted_tokens} tokens "
|
||||
f"({result.messages_removed} messages removed, strategy: {policy.strategy.value})"
|
||||
)
|
||||
|
||||
|
||||
# Issue #2741/#3062: persist the summary so async resume is cheap too,
|
||||
# matching the sync proactive-compaction path above. No-op unless a
|
||||
# session store + session_id are bound and a summary was produced.
|
||||
# append_compaction_checkpoint() does a locked read/modify/write to
|
||||
# disk, so offload it to a worker thread to avoid stalling the event
|
||||
# loop (streaming, tool calls, other agents) during the file I/O.
|
||||
try:
|
||||
await asyncio.to_thread(self._persist_compaction_checkpoint, result)
|
||||
except Exception as e:
|
||||
logging.debug(f"Failed to persist compaction checkpoint (async): {e}")
|
||||
|
||||
try:
|
||||
await self._hook_runner.execute(_HookEvent.AFTER_COMPACTION, result)
|
||||
except Exception as e:
|
||||
@@ -4021,7 +4120,14 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
} for tc in tool_calls_data if tc['id']
|
||||
]
|
||||
self._append_to_chat_history(assistant_message)
|
||||
|
||||
# Persist the assistant tool-call turn so resume replays
|
||||
# it (Issue #3089).
|
||||
self._persist_message(
|
||||
"assistant",
|
||||
response_text,
|
||||
tool_calls=assistant_message.get("tool_calls"),
|
||||
)
|
||||
|
||||
# Execute tool calls and add results to chat history.
|
||||
# Media-bearing follow-up messages are deferred until all
|
||||
# tool replies for this turn are appended, keeping the
|
||||
@@ -4058,6 +4164,12 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
"tool_call_id": tool_call['id'],
|
||||
"content": str(tool_result)
|
||||
})
|
||||
# Persist the tool-result turn (Issue #3089).
|
||||
self._persist_message(
|
||||
"tool",
|
||||
str(tool_result),
|
||||
tool_call_id=tool_call['id'],
|
||||
)
|
||||
except Exception as tool_error:
|
||||
logging.error(f"Tool execution error in streaming: {tool_error}")
|
||||
# Add error result to chat history
|
||||
@@ -4066,6 +4178,11 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
"tool_call_id": tool_call['id'],
|
||||
"content": f"Error: {str(tool_error)}"
|
||||
})
|
||||
self._persist_message(
|
||||
"tool",
|
||||
f"Error: {str(tool_error)}",
|
||||
tool_call_id=tool_call['id'],
|
||||
)
|
||||
|
||||
# Flush deferred media follow-ups after all tool replies.
|
||||
for _m in _deferred_media_followups:
|
||||
|
||||
@@ -20,10 +20,14 @@ from ._lazy_display import _get_console, _get_live, _get_display_functions
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
||||
from typing import List, Optional, Any, Dict, Union, Generator, TYPE_CHECKING
|
||||
|
||||
# Shared HTTP launch state (Agent.launch() multi-endpoint registration)
|
||||
_server_lock = threading.Lock()
|
||||
_server_started: Dict[int, bool] = {}
|
||||
_registered_agents: Dict[int, Dict[str, str]] = {}
|
||||
_shared_apps: Dict[int, Any] = {}
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
@@ -742,7 +746,7 @@ Write the complete compiled report:"""
|
||||
|
||||
# Show animated status during LLM call if verbose
|
||||
if self.verbose and is_tty:
|
||||
from ..main import PRAISON_COLORS, sync_display_callbacks
|
||||
from ..main import PRAISON_COLORS, sync_display_callbacks, _callbacks_lock
|
||||
import threading
|
||||
import time as time_module
|
||||
|
||||
@@ -763,9 +767,11 @@ Write the complete compiled report:"""
|
||||
tools_called.append(tool_name)
|
||||
current_status[0] = f"Calling tool: {tool_name}..."
|
||||
|
||||
# Store original callback and register ours
|
||||
original_tool_callback = sync_display_callbacks.get('tool_call')
|
||||
sync_display_callbacks['tool_call'] = status_tool_callback
|
||||
# Store original callback and register ours (use the module's
|
||||
# own lock so this doesn't race concurrent verbose agents)
|
||||
with _callbacks_lock:
|
||||
original_tool_callback = sync_display_callbacks.get('tool_call')
|
||||
sync_display_callbacks['tool_call'] = status_tool_callback
|
||||
|
||||
# Animation state
|
||||
result_holder = [None]
|
||||
@@ -803,11 +809,16 @@ Write the complete compiled report:"""
|
||||
error_holder[0] = e
|
||||
finally:
|
||||
self.verbose = original_verbose_chat
|
||||
# Restore original callback
|
||||
if original_tool_callback:
|
||||
sync_display_callbacks['tool_call'] = original_tool_callback
|
||||
elif 'tool_call' in sync_display_callbacks:
|
||||
del sync_display_callbacks['tool_call']
|
||||
# Restore original callback under the lock. The identity
|
||||
# check guards the entire restore, so we only mutate the
|
||||
# entry while it's still ours and never clobber a callback
|
||||
# a concurrent verbose agent has since installed.
|
||||
with _callbacks_lock:
|
||||
if sync_display_callbacks.get('tool_call') is status_tool_callback:
|
||||
if original_tool_callback:
|
||||
sync_display_callbacks['tool_call'] = original_tool_callback
|
||||
else:
|
||||
del sync_display_callbacks['tool_call']
|
||||
|
||||
# Start chat in background thread
|
||||
chat_thread = threading.Thread(target=run_chat)
|
||||
@@ -985,6 +996,83 @@ Write the complete compiled report:"""
|
||||
"""Backward-compatible async alias for :meth:`alearn_skill`."""
|
||||
return await self.alearn_skill(request, **kwargs)
|
||||
|
||||
def optimize_instructions(
|
||||
self,
|
||||
evalset: List[Any],
|
||||
*,
|
||||
metric: Optional[Any] = None,
|
||||
scorer: Optional[Any] = None,
|
||||
criteria: str = "",
|
||||
n_candidates: int = 6,
|
||||
apply: bool = True,
|
||||
) -> Any:
|
||||
"""Optimise this agent's own ``instructions`` against an eval set.
|
||||
|
||||
Opt-in, off by default. Generates ``n_candidates`` instruction variants,
|
||||
scores each over ``evalset`` (LLM ``Judge`` by default, or a numeric
|
||||
``metric`` you supply), keeps the highest-scoring one, and — when
|
||||
``apply=True`` — writes it back to ``self.instructions``.
|
||||
|
||||
Args:
|
||||
evalset: List of ``(prompt, expected)`` cases to score candidates on.
|
||||
metric: Optional numeric metric ``(output, expected) -> float``.
|
||||
When set, empirical scoring replaces the LLM Judge.
|
||||
scorer: Optional custom ``Judge`` instance (ignored if ``metric`` set).
|
||||
criteria: Optional criteria for the default Judge.
|
||||
n_candidates: Number of instruction variants to try (default: 6).
|
||||
apply: Write the winning instructions back to the agent (default: True).
|
||||
|
||||
Returns:
|
||||
``OptimizeResult`` with ``best_instructions``, ``best_score``,
|
||||
``base_score`` and the full ``trials`` list.
|
||||
|
||||
Example::
|
||||
|
||||
result = agent.optimize_instructions(
|
||||
evalset=[("summarise X", gold_x)], metric=rouge_l,
|
||||
)
|
||||
print(result.best_score, result.best_instructions)
|
||||
"""
|
||||
from praisonaiagents.eval.prompt_optimizer import PromptOptimizer
|
||||
|
||||
return PromptOptimizer(
|
||||
self,
|
||||
evalset,
|
||||
scorer=scorer,
|
||||
metric=metric,
|
||||
criteria=criteria,
|
||||
n_candidates=n_candidates,
|
||||
apply=apply,
|
||||
).optimize()
|
||||
|
||||
async def aoptimize_instructions(
|
||||
self,
|
||||
evalset: List[Any],
|
||||
*,
|
||||
metric: Optional[Any] = None,
|
||||
scorer: Optional[Any] = None,
|
||||
criteria: str = "",
|
||||
n_candidates: int = 6,
|
||||
apply: bool = True,
|
||||
) -> Any:
|
||||
"""Async twin of :meth:`optimize_instructions`.
|
||||
|
||||
The optimiser runs synchronous agent calls internally; this variant
|
||||
offloads the run to a worker thread so async callers never block the
|
||||
event loop.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
return await asyncio.to_thread(
|
||||
self.optimize_instructions,
|
||||
evalset,
|
||||
metric=metric,
|
||||
scorer=scorer,
|
||||
criteria=criteria,
|
||||
n_candidates=n_candidates,
|
||||
apply=apply,
|
||||
)
|
||||
|
||||
def _ensure_skill_management_tools(self) -> None:
|
||||
"""Ensure the agent has the ``skill_manage`` tool for authoring skills.
|
||||
|
||||
@@ -1046,7 +1134,14 @@ Write the complete compiled report:"""
|
||||
return await self.achat(prompt, task_name=task_name, task_description=task_description, task_id=task_id)
|
||||
|
||||
async def execute_tool_async(self, function_name: str, arguments: Dict[str, Any], tool_call_id: Optional[str] = None, tools_override: Optional[List] = None) -> Any:
|
||||
"""Async version of execute_tool with retry policy support"""
|
||||
"""Async version of execute_tool with retry policy support.
|
||||
|
||||
Routes through the user-supplied tool middleware (``Agent(hooks=[...])``)
|
||||
when present so ``before_tool``/``after_tool``/``wrap_tool_call`` gate
|
||||
async tool calls exactly as they do sync ones (security parity). The
|
||||
fast path (no tool hooks) calls straight into the retry loop with zero
|
||||
overhead.
|
||||
"""
|
||||
|
||||
# Record the tool name for this turn so the self-improve review policy
|
||||
# sees async tool usage too (issue #3037). Mirrors the sync path in
|
||||
@@ -1059,6 +1154,125 @@ Write the complete compiled report:"""
|
||||
turn_tools = self._turn_tools_used
|
||||
turn_tools.append(function_name)
|
||||
|
||||
# Enforce BEFORE_TOOL/AFTER_TOOL security hooks for every async caller,
|
||||
# mirroring the sync execute_tool path in tool_execution.py. Without
|
||||
# this the primary async path (_execute_unified_achat_completion) would
|
||||
# silently skip HookEvent.BEFORE_TOOL gating registered via
|
||||
# Agent(hooks=HookRegistry(...)). Zero overhead when no hooks apply.
|
||||
hook_runner = getattr(self, '_hook_runner', None)
|
||||
if hook_runner is not None:
|
||||
from ..hooks import HookEvent
|
||||
if hook_runner.registry.has_hooks(HookEvent.BEFORE_TOOL):
|
||||
from ..hooks import BeforeToolInput
|
||||
before_tool_input = BeforeToolInput(
|
||||
session_id=getattr(self, '_session_id', 'default'),
|
||||
cwd=os.getcwd(),
|
||||
event_name=HookEvent.BEFORE_TOOL,
|
||||
timestamp=str(time.time()),
|
||||
agent_name=self.name,
|
||||
tool_name=function_name,
|
||||
tool_input=arguments,
|
||||
)
|
||||
before_results = await hook_runner.execute(
|
||||
HookEvent.BEFORE_TOOL, before_tool_input, target=function_name
|
||||
)
|
||||
if hook_runner.is_blocked(before_results):
|
||||
logging.warning(f"Tool {function_name} execution blocked by BEFORE_TOOL hook")
|
||||
return f"Execution of {function_name} was blocked by security policy."
|
||||
for res in before_results:
|
||||
if res.output and res.output.modified_input:
|
||||
arguments.update(res.output.modified_input)
|
||||
|
||||
result = await self._execute_tool_async_dispatch(
|
||||
function_name, arguments, tool_call_id, tools_override
|
||||
)
|
||||
|
||||
if hook_runner.registry.has_hooks(HookEvent.AFTER_TOOL):
|
||||
from ..hooks import AfterToolInput
|
||||
after_tool_input = AfterToolInput(
|
||||
session_id=getattr(self, '_session_id', 'default'),
|
||||
cwd=os.getcwd(),
|
||||
event_name=HookEvent.AFTER_TOOL,
|
||||
timestamp=str(time.time()),
|
||||
agent_name=self.name,
|
||||
tool_name=function_name,
|
||||
tool_input=arguments,
|
||||
tool_output=result,
|
||||
)
|
||||
after_results = await hook_runner.execute(
|
||||
HookEvent.AFTER_TOOL, after_tool_input, target=function_name
|
||||
)
|
||||
extra_context = hook_runner.aggregate_context(after_results)
|
||||
if extra_context:
|
||||
if isinstance(result, str):
|
||||
result = f"{result}\n\n{extra_context}"
|
||||
elif isinstance(result, dict):
|
||||
result.setdefault("_additional_context", extra_context)
|
||||
return result
|
||||
|
||||
return await self._execute_tool_async_dispatch(
|
||||
function_name, arguments, tool_call_id, tools_override
|
||||
)
|
||||
|
||||
async def _execute_tool_async_dispatch(self, function_name: str, arguments: Dict[str, Any], tool_call_id: Optional[str] = None, tools_override: Optional[List] = None) -> Any:
|
||||
"""Route an async tool call through the middleware chain (if any) then retry loop."""
|
||||
# Route async tool calls through the same tool middleware chain as the
|
||||
# sync path. The chain (before_tool/after_tool/wrap_tool_call) is
|
||||
# synchronous, so we run it in a worker thread and bridge its final
|
||||
# handler back to this event loop via run_coroutine_threadsafe.
|
||||
manager = self._get_tool_middleware_manager()
|
||||
if manager is not None:
|
||||
return await self._execute_tool_async_via_middleware(
|
||||
manager, function_name, arguments, tool_call_id, tools_override
|
||||
)
|
||||
return await self._execute_tool_async_with_retry(
|
||||
function_name, arguments, tool_call_id, tools_override
|
||||
)
|
||||
|
||||
async def _execute_tool_async_via_middleware(
|
||||
self, manager, function_name, arguments, tool_call_id, tools_override
|
||||
):
|
||||
"""Drive the (sync) tool middleware chain around an async tool call.
|
||||
|
||||
The middleware manager and its ``wrap_tool_call`` chain are synchronous
|
||||
by contract, so they run in a thread-pool worker; the innermost handler
|
||||
schedules the actual async execution back on the current running loop
|
||||
and blocks the worker on the result. This preserves short-circuit,
|
||||
audit, and argument-mutation semantics for async tools.
|
||||
"""
|
||||
from ..hooks import ToolRequest, ToolResponse, InvocationContext
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
request = ToolRequest(
|
||||
tool_name=function_name,
|
||||
arguments=arguments,
|
||||
context=InvocationContext(
|
||||
agent_id=self.name,
|
||||
run_id=getattr(self, '_current_run_id', 'unknown'),
|
||||
session_id=getattr(self, '_session_id', None) or 'default',
|
||||
tool_name=function_name,
|
||||
),
|
||||
)
|
||||
|
||||
def _final_handler(req):
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
self._execute_tool_async_with_retry(
|
||||
req.tool_name, req.arguments, tool_call_id, tools_override
|
||||
),
|
||||
loop,
|
||||
)
|
||||
result = future.result()
|
||||
return ToolResponse(tool_name=req.tool_name, result=result)
|
||||
|
||||
def _run_chain():
|
||||
return manager.execute_tool_call(request, _final_handler)
|
||||
|
||||
response = await loop.run_in_executor(None, _run_chain)
|
||||
return response.result if isinstance(response, ToolResponse) else response
|
||||
|
||||
async def _execute_tool_async_with_retry(self, function_name: str, arguments: Dict[str, Any], tool_call_id: Optional[str] = None, tools_override: Optional[List] = None) -> Any:
|
||||
"""Async tool execution with retry policy (middleware-agnostic core)."""
|
||||
|
||||
# Get retry policy (tool-level > agent-level > default)
|
||||
retry_policy = self._get_tool_retry_policy(function_name)
|
||||
|
||||
@@ -1144,7 +1358,17 @@ Write the complete compiled report:"""
|
||||
error_msg = f"Error during approval process: {str(e)}"
|
||||
logging.error(error_msg)
|
||||
return {"error": error_msg, "approval_error": True}
|
||||
|
||||
|
||||
# Policy/guardrail gate (protocol-driven). Mirrors the sync path in
|
||||
# _execute_tool_impl so async callers cannot bypass a PolicyEngine
|
||||
# deny or a tool-call guardrail. The check is pure/sync (no awaits).
|
||||
check = getattr(self, "_check_tool_policy_and_guardrails", None)
|
||||
if check is not None:
|
||||
policy_result = check(function_name, arguments)
|
||||
if isinstance(policy_result, dict):
|
||||
return policy_result # Error dict
|
||||
_, arguments = policy_result
|
||||
|
||||
# Try to find the function in the override tools list first, then agent's tools list
|
||||
func = None
|
||||
tools_to_search = tools_override if tools_override is not None else self.tools
|
||||
@@ -1349,6 +1573,11 @@ Write the complete compiled report:"""
|
||||
except Exception as e:
|
||||
logging.error(f"Error starting server: {str(e)}", exc_info=True)
|
||||
print(f"❌ Error starting server: {str(e)}")
|
||||
finally:
|
||||
# Clear the started marker so a later launch() on this port
|
||||
# can retry after a bind/startup failure or server exit.
|
||||
with _server_lock:
|
||||
_server_started[port] = False
|
||||
|
||||
# Run server in a background thread
|
||||
server_thread = threading.Thread(target=run_server, daemon=True)
|
||||
|
||||
@@ -7,6 +7,7 @@ for maintainability.
|
||||
"""
|
||||
|
||||
import os
|
||||
import inspect
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
@@ -367,8 +368,25 @@ class MemoryMixin:
|
||||
|
||||
self._current_run_id = None
|
||||
|
||||
def _persist_message(self, role: str, content: str):
|
||||
"""Persist a message to the DB or session store."""
|
||||
def _persist_message(
|
||||
self,
|
||||
role: str,
|
||||
content: str,
|
||||
tool_calls=None,
|
||||
tool_call_id: Optional[str] = None,
|
||||
):
|
||||
"""Persist a message to the DB or session store.
|
||||
|
||||
Args:
|
||||
role: Message role ("user", "assistant", "system", "tool").
|
||||
content: Message text.
|
||||
tool_calls: Optional structured tool calls on an assistant turn
|
||||
(Issue #3089). Persisted faithfully to the default JSON store
|
||||
so a resumed tool-using session reconstructs the same message
|
||||
list the model saw before.
|
||||
tool_call_id: Optional id linking a ``role="tool"`` result turn to
|
||||
the assistant tool call it answers (Issue #3089).
|
||||
"""
|
||||
# Try DB adapter first
|
||||
if self._db is not None:
|
||||
try:
|
||||
@@ -386,7 +404,23 @@ class MemoryMixin:
|
||||
if role == "user":
|
||||
self._session_store.add_user_message(self._session_id, content)
|
||||
elif role == "assistant":
|
||||
self._session_store.add_assistant_message(self._session_id, content)
|
||||
# Faithful transcript: carry any tool calls the assistant
|
||||
# requested so resume replays them (Issue #3089). Falls back
|
||||
# to a plain text turn for stores predating the tool fields.
|
||||
if tool_calls:
|
||||
self._add_message_with_tool_fields(
|
||||
"assistant", content, tool_calls=tool_calls
|
||||
)
|
||||
else:
|
||||
self._session_store.add_assistant_message(
|
||||
self._session_id, content
|
||||
)
|
||||
elif role == "tool":
|
||||
# Persist the tool-result turn linked to its call id so the
|
||||
# resumed message list interleaves results in order (#3089).
|
||||
self._add_message_with_tool_fields(
|
||||
"tool", content, tool_call_id=tool_call_id
|
||||
)
|
||||
# Keep auto_save index in sync when per-turn persist shares session_id
|
||||
if self.auto_save and self.auto_save == self._session_id:
|
||||
with self._history_lock:
|
||||
@@ -397,6 +431,45 @@ class MemoryMixin:
|
||||
except Exception as e:
|
||||
logging.warning(f"Failed to persist message to session store: {e}")
|
||||
|
||||
def _add_message_with_tool_fields(self, role, content, **tool_fields):
|
||||
"""Add a message carrying tool fields, tolerating stores without them.
|
||||
|
||||
The default JSON store and the SQLite store accept ``tool_calls`` /
|
||||
``tool_call_id``, but custom stores implementing the older
|
||||
``SessionStoreProtocol`` (and the built-in ``HierarchicalSessionStore``)
|
||||
only accept ``(session_id, role, content, metadata)``. Passing the new
|
||||
keywords to those would raise ``TypeError`` and drop the turn entirely
|
||||
(Issue #3089). We feature-detect once via the call signature and fall
|
||||
back to a plain-text turn so no exchange is silently lost.
|
||||
"""
|
||||
add_message = self._session_store.add_message
|
||||
if self._store_supports_tool_fields(add_message):
|
||||
add_message(self._session_id, role, content, **tool_fields)
|
||||
return
|
||||
# Legacy store: preserve the turn as plain text so resume still sees it.
|
||||
add_message(self._session_id, role, content)
|
||||
|
||||
def _store_supports_tool_fields(self, add_message) -> bool:
|
||||
"""Return True if ``add_message`` accepts the tool keyword fields."""
|
||||
cached = getattr(self, "_session_store_tool_fields", None)
|
||||
if cached is not None:
|
||||
return cached
|
||||
supported = True
|
||||
try:
|
||||
params = inspect.signature(add_message).parameters
|
||||
has_var_kw = any(
|
||||
p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()
|
||||
)
|
||||
supported = has_var_kw or (
|
||||
"tool_calls" in params and "tool_call_id" in params
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
# Builtins / C-callables without introspectable signatures: assume
|
||||
# the conservative legacy shape.
|
||||
supported = False
|
||||
self._session_store_tool_fields = supported
|
||||
return supported
|
||||
|
||||
def _persist_session_stats(self):
|
||||
"""Flush agent cost/token stats into session JSON metadata."""
|
||||
store = getattr(self, "_session_store", None)
|
||||
|
||||
@@ -426,8 +426,50 @@ class ToolExecutionMixin:
|
||||
metadata={'agent_name': self.name}
|
||||
)
|
||||
|
||||
# Execute within injection context
|
||||
return self._execute_tool_with_context(function_name, arguments, state, tool_call_id)
|
||||
# Route through user-supplied tool middleware (Agent(hooks=[...])) when
|
||||
# present. Zero overhead when no hooks: the fast path calls straight
|
||||
# into _execute_tool_with_context.
|
||||
manager = self._get_tool_middleware_manager()
|
||||
if manager is None:
|
||||
return self._execute_tool_with_context(function_name, arguments, state, tool_call_id)
|
||||
|
||||
from ..hooks import ToolRequest, ToolResponse, InvocationContext
|
||||
request = ToolRequest(
|
||||
tool_name=function_name,
|
||||
arguments=arguments,
|
||||
context=InvocationContext(
|
||||
agent_id=self.name,
|
||||
run_id=getattr(self, '_current_run_id', 'unknown'),
|
||||
session_id=getattr(self, '_session_id', None) or 'default',
|
||||
tool_name=function_name,
|
||||
),
|
||||
)
|
||||
|
||||
def _final_handler(req: ToolRequest) -> ToolResponse:
|
||||
result = self._execute_tool_with_context(
|
||||
req.tool_name, req.arguments, state, tool_call_id
|
||||
)
|
||||
return ToolResponse(tool_name=req.tool_name, result=result)
|
||||
|
||||
response = manager.execute_tool_call(request, _final_handler)
|
||||
return response.result if isinstance(response, ToolResponse) else response
|
||||
|
||||
def _get_tool_middleware_manager(self):
|
||||
"""Return a MiddlewareManager if user tool hooks are registered, else None.
|
||||
|
||||
Lazily constructs the manager from ``self._hooks`` (the list passed via
|
||||
``Agent(hooks=[...])``) on first use. Returns ``None`` when there are no
|
||||
hooks or no tool-level hooks, preserving the zero-overhead fast path.
|
||||
"""
|
||||
hooks = getattr(self, '_hooks', None)
|
||||
if not hooks:
|
||||
return None
|
||||
manager = getattr(self, '_middleware_manager', None)
|
||||
if manager is None:
|
||||
from ..hooks import MiddlewareManager
|
||||
manager = MiddlewareManager(hooks)
|
||||
self._middleware_manager = manager
|
||||
return manager if manager.has_tool_hooks else None
|
||||
|
||||
def _execute_tool_with_context(self, function_name, arguments, state, tool_call_id=None):
|
||||
"""Execute tool within injection context, with optional output truncation.
|
||||
@@ -685,7 +727,7 @@ class ToolExecutionMixin:
|
||||
# For other error dicts: approval/permission denials are legitimate
|
||||
# non-retryable outcomes; everything else represents a tool failure
|
||||
# that should engage the outer retry/backoff loop.
|
||||
elif result.get("approval_denied") or result.get("permission_denied") or result.get("approval_error"):
|
||||
elif result.get("approval_denied") or result.get("permission_denied") or result.get("approval_error") or result.get("policy_denied") or result.get("guardrail_denied"):
|
||||
break
|
||||
else:
|
||||
# Avoid compounding with the inner retry loop in
|
||||
@@ -987,10 +1029,15 @@ class ToolExecutionMixin:
|
||||
if hasattr(self, '_ensure_loop_guard'):
|
||||
loop_guard = self._ensure_loop_guard()
|
||||
is_success = result is not None and not (isinstance(result, dict) and result.get('error'))
|
||||
loop_guard.record(function_name, arguments, is_success)
|
||||
# Handle warning injection for WARN decisions
|
||||
decision = loop_guard.check(function_name, arguments, is_pre_execution=False)
|
||||
if decision.action.value == "warn":
|
||||
loop_guard.record(function_name, arguments, is_success, result=result)
|
||||
# Surface the loop-guard decision back to the model on this same
|
||||
# iteration. Previously only WARN was injected, so a post-exec
|
||||
# BLOCK/HALT (e.g. the call that first reaches a threshold) was
|
||||
# silently discarded and only took effect on the next
|
||||
# pre-execution check. Injecting block/halt here ensures the stop
|
||||
# signal reaches the model immediately without raising mid-turn.
|
||||
decision = loop_guard.check(function_name, arguments, is_pre_execution=False)
|
||||
if decision.action.value in ("warn", "block", "halt"):
|
||||
if isinstance(result, str):
|
||||
result = f"{result}\n\n[loop-guard] {decision.message}"
|
||||
elif isinstance(result, dict):
|
||||
@@ -1712,6 +1759,8 @@ class ToolExecutionMixin:
|
||||
if (result.get("approval_denied") or
|
||||
result.get("permission_denied") or
|
||||
result.get("approval_error") or
|
||||
result.get("policy_denied") or
|
||||
result.get("guardrail_denied") or
|
||||
result.get("circuit_open")):
|
||||
return result
|
||||
|
||||
@@ -1810,8 +1859,10 @@ class ToolExecutionMixin:
|
||||
|
||||
try:
|
||||
|
||||
# Get or create circuit breaker for this tool
|
||||
breaker_name = f"tool_{function_name}"
|
||||
# Get or create circuit breaker for this tool.
|
||||
# Scope the key to this Agent instance so one agent's failing tool
|
||||
# cannot trip the breaker for another agent's same-named tool.
|
||||
breaker_name = f"tool_{id(self)}_{function_name}"
|
||||
config = CircuitBreakerConfig(
|
||||
failure_threshold=5, # Open after 5 failures
|
||||
recovery_timeout=60.0, # Wait 60s before trying half-open
|
||||
@@ -1828,7 +1879,9 @@ class ToolExecutionMixin:
|
||||
if isinstance(result, dict) and result.get("error") and \
|
||||
not result.get("approval_denied") and \
|
||||
not result.get("permission_denied") and \
|
||||
not result.get("approval_error"):
|
||||
not result.get("approval_error") and \
|
||||
not result.get("policy_denied") and \
|
||||
not result.get("guardrail_denied"):
|
||||
# Create a sentinel exception to register failure with circuit breaker
|
||||
class _ToolFailure(Exception):
|
||||
def __init__(self, error_dict):
|
||||
@@ -1857,6 +1910,68 @@ class ToolExecutionMixin:
|
||||
"remediation": "Wait for recovery_timeout (60s) or investigate recent tool failures.",
|
||||
}
|
||||
|
||||
def _check_tool_policy_and_guardrails(self, function_name, arguments):
|
||||
"""Gate a tool call through the attached PolicyEngine and tool guardrails.
|
||||
|
||||
Consults ``self._policy`` (a ``PolicyEngine``) via ``check_tool`` and any
|
||||
tool-call guardrails exposing ``validate_tool_call``. Returns an error
|
||||
dict when the call is denied, or ``(None, arguments)`` (arguments possibly
|
||||
rewritten by a guardrail) when allowed. Zero overhead when neither is set.
|
||||
"""
|
||||
policy = getattr(self, "_policy", None)
|
||||
if policy is not None and hasattr(policy, "check_tool"):
|
||||
try:
|
||||
result = policy.check_tool(function_name, arguments)
|
||||
except Exception as e: # noqa: BLE001
|
||||
# Fail closed: an operator opted into policy enforcement, so a
|
||||
# broken/misconfigured PolicyEngine must deny rather than let a
|
||||
# protected tool run without a decision.
|
||||
logging.warning(
|
||||
f"Tool '{function_name}' denied: policy check_tool raised: {e}"
|
||||
)
|
||||
return {
|
||||
"error": f"Tool '{function_name}' denied: policy check failed ({e})",
|
||||
"policy_denied": True,
|
||||
}
|
||||
if not getattr(result, "allowed", True):
|
||||
reason = getattr(result, "reason", "denied by policy")
|
||||
logging.warning(
|
||||
f"Tool '{function_name}' denied by policy: {reason}"
|
||||
)
|
||||
return {
|
||||
"error": f"Tool '{function_name}' denied by policy: {reason}",
|
||||
"policy_denied": True,
|
||||
}
|
||||
|
||||
for guardrail in getattr(self, "_tool_call_guardrails", None) or []:
|
||||
validate = getattr(guardrail, "validate_tool_call", None)
|
||||
if validate is None:
|
||||
continue
|
||||
try:
|
||||
is_valid, processed = validate(function_name, arguments)
|
||||
except Exception as e: # noqa: BLE001
|
||||
# Fail closed: mirror the guardrail-chain default. A guardrail
|
||||
# dependency/implementation error must block, not permit, the
|
||||
# unchecked call.
|
||||
logging.warning(
|
||||
f"Tool '{function_name}' denied: guardrail validate_tool_call raised: {e}"
|
||||
)
|
||||
return {
|
||||
"error": f"Tool '{function_name}' denied: guardrail check failed ({e})",
|
||||
"guardrail_denied": True,
|
||||
}
|
||||
if not is_valid:
|
||||
logging.warning(
|
||||
f"Tool '{function_name}' rejected by tool-call guardrail"
|
||||
)
|
||||
return {
|
||||
"error": f"Tool '{function_name}' rejected by guardrail",
|
||||
"guardrail_denied": True,
|
||||
}
|
||||
if isinstance(processed, dict):
|
||||
arguments = processed
|
||||
return None, arguments
|
||||
|
||||
def _execute_tool_impl(self, function_name, arguments):
|
||||
"""Internal tool execution implementation."""
|
||||
|
||||
@@ -1871,6 +1986,14 @@ class ToolExecutionMixin:
|
||||
logging.error(error_msg)
|
||||
return {"error": error_msg, "approval_error": True}
|
||||
|
||||
# Policy/guardrail gate (protocol-driven). Runs after approval so an
|
||||
# explicit PolicyEngine deny or a tool-call guardrail can block a tool
|
||||
# before dispatch (native + MCP, uniform). Zero overhead when unset.
|
||||
policy_result = self._check_tool_policy_and_guardrails(function_name, arguments)
|
||||
if isinstance(policy_result, dict):
|
||||
return policy_result # Error dict
|
||||
_, arguments = policy_result
|
||||
|
||||
# Special handling for MCP tools
|
||||
# Check if tools is an MCP instance with the requested function name
|
||||
MCP = None
|
||||
|
||||
@@ -357,8 +357,12 @@ def _build_execution_context(agents_instance, task_id):
|
||||
"Set task.agent or provide task.agent_config before execution."
|
||||
)
|
||||
|
||||
# Set current agent for token tracking
|
||||
llm = getattr(executor_agent, 'llm', None) or getattr(executor_agent, 'llm_instance', None)
|
||||
# Set current agent for token tracking.
|
||||
# Prefer the real LLM instance (executor_agent.llm is always a model-name
|
||||
# string, so it never exposes set_current_agent/last_token_metrics).
|
||||
llm = getattr(executor_agent, 'llm_instance', None)
|
||||
if llm is None:
|
||||
llm = getattr(executor_agent, 'llm', None)
|
||||
if llm and hasattr(llm, 'set_current_agent'):
|
||||
llm.set_current_agent(executor_agent.display_name)
|
||||
|
||||
@@ -605,7 +609,9 @@ class AgentTeam(SpawnAnnounceProtocol):
|
||||
Args:
|
||||
agents: List of Agent instances
|
||||
tasks: Optional list of Task instances (auto-generated from agents if None)
|
||||
process: Execution process type ("sequential", "parallel", "hierarchical")
|
||||
process: Execution process type ("sequential", "workflow", "hierarchical").
|
||||
For parallel fan-out, set async_execution=True on individual Task
|
||||
objects within a "workflow" or "sequential" process.
|
||||
manager_llm: LLM model for manager agent
|
||||
llm: Default LLM model for all agents
|
||||
name: Name for this agent collection
|
||||
@@ -812,6 +818,14 @@ class AgentTeam(SpawnAnnounceProtocol):
|
||||
self._state_lock = threading.Lock() # Thread-safe state mutations
|
||||
self.verbose = _verbose
|
||||
self.max_retries = _max_retries
|
||||
_VALID_PROCESSES = {"workflow", "sequential", "hierarchical"}
|
||||
if process not in _VALID_PROCESSES:
|
||||
raise ValueError(
|
||||
f"Unknown process type {process!r}. Valid values are: "
|
||||
f"{sorted(_VALID_PROCESSES)}. Note: parallel fan-out is achieved "
|
||||
f"by setting async_execution=True on individual Task objects within "
|
||||
f"a 'workflow' or 'sequential' process, not via process=\"parallel\"."
|
||||
)
|
||||
self.process = process
|
||||
self.stream = _stream
|
||||
self.name = name
|
||||
@@ -1111,6 +1125,63 @@ class AgentTeam(SpawnAnnounceProtocol):
|
||||
logger.warning(f"Task {task_id}: Guardrail processing error (retry {task.retry_count}/{task.max_retries}): {e}")
|
||||
return task_output, True # Signal retry needed
|
||||
|
||||
def _run_task_start_hook(self, task, task_id):
|
||||
"""Run the on_task_start hook and propagate global variables to the task.
|
||||
|
||||
Shared by run_task (sync) and arun_task (async) so the lifecycle hooks
|
||||
and variable propagation stay consistent across both paths.
|
||||
"""
|
||||
if self.on_task_start:
|
||||
try:
|
||||
self.on_task_start(task, task_id)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in on_task_start callback: {e}")
|
||||
|
||||
# Apply global variables to task if not already set. Use a shallow copy so a
|
||||
# task mutating its variables doesn't leak back into the shared AgentTeam state.
|
||||
if self.variables and not getattr(task, 'variables', None):
|
||||
task.variables = self.variables.copy()
|
||||
|
||||
def _run_task_complete_hook(self, task, task_output):
|
||||
"""Run the on_task_complete hook. Shared by run_task and arun_task."""
|
||||
if self.on_task_complete:
|
||||
try:
|
||||
self.on_task_complete(task, task_output)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in on_task_complete callback: {e}")
|
||||
|
||||
async def _arun_task_start_hook(self, task, task_id):
|
||||
"""Async-aware on_task_start hook for arun_task.
|
||||
|
||||
Awaits coroutine callbacks and offloads synchronous ones to the executor so
|
||||
a blocking hook can't stall the event loop. Falls back to the sync helper's
|
||||
variable propagation.
|
||||
"""
|
||||
if self.on_task_start:
|
||||
try:
|
||||
if asyncio.iscoroutinefunction(self.on_task_start):
|
||||
await self.on_task_start(task, task_id)
|
||||
else:
|
||||
loop = asyncio.get_running_loop()
|
||||
await loop.run_in_executor(None, self.on_task_start, task, task_id)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in on_task_start callback: {e}")
|
||||
|
||||
if self.variables and not getattr(task, 'variables', None):
|
||||
task.variables = self.variables.copy()
|
||||
|
||||
async def _arun_task_complete_hook(self, task, task_output):
|
||||
"""Async-aware on_task_complete hook for arun_task (see _arun_task_start_hook)."""
|
||||
if self.on_task_complete:
|
||||
try:
|
||||
if asyncio.iscoroutinefunction(self.on_task_complete):
|
||||
await self.on_task_complete(task, task_output)
|
||||
else:
|
||||
loop = asyncio.get_running_loop()
|
||||
await loop.run_in_executor(None, self.on_task_complete, task, task_output)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in on_task_complete callback: {e}")
|
||||
|
||||
|
||||
async def arun_task(self, task_id):
|
||||
"""Async version of run_task method"""
|
||||
@@ -1122,6 +1193,9 @@ class AgentTeam(SpawnAnnounceProtocol):
|
||||
logger.info(f"Task with ID {task_id} is already completed")
|
||||
return
|
||||
|
||||
# Call on_task_start callback and propagate variables (async-aware, mirrors run_task)
|
||||
await self._arun_task_start_hook(task, task_id)
|
||||
|
||||
# Use per-task max_retries if available
|
||||
task_max = getattr(task, "max_retries", self.max_retries)
|
||||
retries = 0
|
||||
@@ -1150,6 +1224,10 @@ class AgentTeam(SpawnAnnounceProtocol):
|
||||
raise
|
||||
|
||||
self.save_output_to_file(task, task_output)
|
||||
|
||||
# Call on_task_complete callback (async-aware, mirrors run_task)
|
||||
await self._arun_task_complete_hook(task, task_output)
|
||||
|
||||
if self.verbose >= 1:
|
||||
logger.info(f"Task {task_id} completed successfully.")
|
||||
else:
|
||||
@@ -1345,17 +1423,9 @@ class AgentTeam(SpawnAnnounceProtocol):
|
||||
logger.info(f"Task with ID {task_id} is already completed")
|
||||
return
|
||||
|
||||
# Call on_task_start callback if provided
|
||||
if self.on_task_start:
|
||||
try:
|
||||
self.on_task_start(task, task_id)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in on_task_start callback: {e}")
|
||||
|
||||
# Apply global variables to task if not already set
|
||||
if self.variables and not getattr(task, 'variables', None):
|
||||
task.variables = self.variables
|
||||
|
||||
# Call on_task_start callback and propagate variables (shared with arun_task)
|
||||
self._run_task_start_hook(task, task_id)
|
||||
|
||||
# Use per-task max_retries if available
|
||||
task_max = getattr(task, "max_retries", self.max_retries)
|
||||
retries = 0
|
||||
@@ -1380,14 +1450,10 @@ class AgentTeam(SpawnAnnounceProtocol):
|
||||
logger.exception(e)
|
||||
|
||||
self.save_output_to_file(task, task_output)
|
||||
|
||||
# Call on_task_complete callback if provided
|
||||
if self.on_task_complete:
|
||||
try:
|
||||
self.on_task_complete(task, task_output)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in on_task_complete callback: {e}")
|
||||
|
||||
|
||||
# Call on_task_complete callback (shared with arun_task)
|
||||
self._run_task_complete_hook(task, task_output)
|
||||
|
||||
if self.verbose >= 1:
|
||||
logger.info(f"Task {task_id} completed successfully.")
|
||||
else:
|
||||
|
||||
@@ -413,6 +413,35 @@ class MessagePresentation:
|
||||
|
||||
return MessagePresentation(blocks=blocks)
|
||||
|
||||
@staticmethod
|
||||
def question(
|
||||
prompt: str,
|
||||
options: List[Any],
|
||||
context: Optional[str] = None,
|
||||
) -> "MessagePresentation":
|
||||
"""Create a structured question presentation with option buttons.
|
||||
|
||||
The symmetric counterpart to :meth:`approval` for non-binary
|
||||
clarifications ("which of these?"). Each option renders as a typed
|
||||
``reply``-action button (via :meth:`PresentationBlock.quick_replies`),
|
||||
so a tap feeds the chosen value straight back into the next agent turn
|
||||
across any channel, reusing the existing reply routing and byte-safe
|
||||
callback encoding — no new store, protocol, or correlation machinery.
|
||||
|
||||
Args:
|
||||
prompt: The question text.
|
||||
options: Choices as ``(label, value)`` pairs or plain strings.
|
||||
context: Optional context information rendered under the prompt.
|
||||
|
||||
Returns:
|
||||
A presentation with the prompt and one reply button per option.
|
||||
"""
|
||||
blocks = [PresentationBlock.make_text(prompt)]
|
||||
if context:
|
||||
blocks.append(PresentationBlock.make_context(context))
|
||||
blocks.append(PresentationBlock.quick_replies(options))
|
||||
return MessagePresentation(blocks=blocks)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PresentationLimits:
|
||||
|
||||
@@ -5,6 +5,7 @@ Implements shadow git repository for file-level checkpointing.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import asyncio
|
||||
import logging
|
||||
from praisonaiagents._logging import get_logger
|
||||
@@ -27,6 +28,17 @@ def _parse_iso_timestamp(timestamp: str) -> datetime:
|
||||
timestamp = timestamp[:-1] + '+00:00'
|
||||
return datetime.fromisoformat(timestamp)
|
||||
|
||||
# Prefix used to encode a step index into a checkpoint message so per-step
|
||||
# checkpoints can be rewound with restore(step=N) without any extra storage.
|
||||
_STEP_TAG_RE = re.compile(r"^\[step-(\d+)\]\s*")
|
||||
|
||||
|
||||
def _extract_step(message: str) -> Optional[int]:
|
||||
"""Return the step index encoded in a checkpoint message, or None."""
|
||||
match = _STEP_TAG_RE.match(message or "")
|
||||
return int(match.group(1)) if match else None
|
||||
|
||||
|
||||
# Protected paths that should never be checkpointed
|
||||
PROTECTED_PATHS = [
|
||||
os.path.expanduser("~"),
|
||||
@@ -249,13 +261,21 @@ class CheckpointService:
|
||||
|
||||
return env
|
||||
|
||||
async def save(self, message: str, allow_empty: bool = False) -> CheckpointResult:
|
||||
async def save(
|
||||
self,
|
||||
message: str,
|
||||
allow_empty: bool = False,
|
||||
step: Optional[int] = None,
|
||||
) -> CheckpointResult:
|
||||
"""
|
||||
Save a checkpoint.
|
||||
|
||||
Args:
|
||||
message: Checkpoint message
|
||||
allow_empty: Allow checkpoint even if no changes
|
||||
step: Optional step index. When provided, the checkpoint is tagged
|
||||
as a per-step checkpoint and can be rewound with
|
||||
``restore(step=...)``.
|
||||
|
||||
Returns:
|
||||
CheckpointResult with the created checkpoint
|
||||
@@ -263,6 +283,16 @@ class CheckpointService:
|
||||
if not self._initialized:
|
||||
return CheckpointResult.fail("Service not initialized")
|
||||
|
||||
if step is not None and step < 0:
|
||||
return CheckpointResult.fail("step must be a non-negative integer")
|
||||
|
||||
# Encode the step index into the message so it can be recovered later
|
||||
# without any extra storage (reuses the shadow-git commit log). An
|
||||
# explicit step always wins: strip any existing tag before prepending
|
||||
# so save("[step-2] retry", step=1) is indexed as step 1.
|
||||
if step is not None:
|
||||
message = f"[step-{step}] {_STEP_TAG_RE.sub('', message)}"
|
||||
|
||||
try:
|
||||
# Stage all changes
|
||||
await self._run_git("add", "-A")
|
||||
@@ -292,7 +322,8 @@ class CheckpointService:
|
||||
id=commit_hash,
|
||||
short_id=commit_hash[:8],
|
||||
message=message,
|
||||
timestamp=_parse_iso_timestamp(timestamp)
|
||||
timestamp=_parse_iso_timestamp(timestamp),
|
||||
step=_extract_step(message)
|
||||
)
|
||||
|
||||
self._checkpoints.append(checkpoint)
|
||||
@@ -310,12 +341,18 @@ class CheckpointService:
|
||||
self._emit(CheckpointEvent.ERROR, {"error": error_msg})
|
||||
return CheckpointResult.fail(error_msg)
|
||||
|
||||
async def restore(self, checkpoint_id: str) -> CheckpointResult:
|
||||
async def restore(
|
||||
self,
|
||||
checkpoint_id: Optional[str] = None,
|
||||
step: Optional[int] = None,
|
||||
) -> CheckpointResult:
|
||||
"""
|
||||
Restore workspace to a checkpoint.
|
||||
|
||||
Args:
|
||||
checkpoint_id: Checkpoint ID (commit hash) to restore
|
||||
step: Restore the per-step checkpoint tagged with this step index.
|
||||
Mutually exclusive with ``checkpoint_id``.
|
||||
|
||||
Returns:
|
||||
CheckpointResult indicating success/failure
|
||||
@@ -323,6 +360,20 @@ class CheckpointService:
|
||||
if not self._initialized:
|
||||
return CheckpointResult.fail("Service not initialized")
|
||||
|
||||
if checkpoint_id is not None and step is not None:
|
||||
return CheckpointResult.fail(
|
||||
"Provide either checkpoint_id or step, not both"
|
||||
)
|
||||
|
||||
if step is not None:
|
||||
checkpoint = await self.get_checkpoint_by_step(step)
|
||||
if checkpoint is None:
|
||||
return CheckpointResult.fail(f"No checkpoint found for step {step}")
|
||||
checkpoint_id = checkpoint.id
|
||||
|
||||
if checkpoint_id is None:
|
||||
return CheckpointResult.fail("No checkpoint id or step provided")
|
||||
|
||||
try:
|
||||
# Clean untracked files
|
||||
await self._run_git("clean", "-f", "-d")
|
||||
@@ -338,7 +389,8 @@ class CheckpointService:
|
||||
id=checkpoint_id,
|
||||
short_id=checkpoint_id[:8],
|
||||
message=message,
|
||||
timestamp=_parse_iso_timestamp(timestamp)
|
||||
timestamp=_parse_iso_timestamp(timestamp),
|
||||
step=_extract_step(message)
|
||||
)
|
||||
|
||||
self._emit(CheckpointEvent.CHECKPOINT_RESTORED, checkpoint)
|
||||
@@ -351,6 +403,47 @@ class CheckpointService:
|
||||
self._emit(CheckpointEvent.ERROR, {"error": error_msg})
|
||||
return CheckpointResult.fail(error_msg)
|
||||
|
||||
async def rewind(self, steps: int = 1) -> CheckpointResult:
|
||||
"""
|
||||
Rewind the workspace back ``steps`` checkpoints from the latest.
|
||||
|
||||
Checkpoints form an ordered sequence (newest first). ``rewind(1)``
|
||||
restores the checkpoint immediately before the current one (undoing the
|
||||
most recent checkpointed change); ``rewind(n)`` steps back ``n``
|
||||
checkpoints.
|
||||
|
||||
Note: a checkpoint is not guaranteed to correspond 1:1 with an agent
|
||||
turn — manual saves and auto-checkpoints both create checkpoints — so
|
||||
``steps`` counts checkpoints, which is the closest turn-addressable
|
||||
primitive available without persisting a turn↔checkpoint map.
|
||||
|
||||
Args:
|
||||
steps: How many checkpoints to step back (must be >= 1).
|
||||
|
||||
Returns:
|
||||
CheckpointResult with the checkpoint restored to.
|
||||
"""
|
||||
if not self._initialized:
|
||||
return CheckpointResult.fail("Service not initialized")
|
||||
|
||||
if steps < 1:
|
||||
return CheckpointResult.fail("steps must be >= 1")
|
||||
|
||||
# Query only as many checkpoints as we need to reach the target.
|
||||
# Shadow-git retains every commit (pruning only trims the in-memory
|
||||
# list), so we must not cap the lookup at ``max_checkpoints`` or valid
|
||||
# older targets would become unreachable.
|
||||
checkpoints = await self.list_checkpoints(limit=steps + 1)
|
||||
if steps >= len(checkpoints):
|
||||
return CheckpointResult.fail(
|
||||
f"Cannot rewind {steps} step(s): only {len(checkpoints)} checkpoint(s) available"
|
||||
)
|
||||
|
||||
# list_checkpoints is newest-first, so index ``steps`` is the checkpoint
|
||||
# ``steps`` positions back from the latest.
|
||||
target = checkpoints[steps]
|
||||
return await self.restore(target.id)
|
||||
|
||||
async def diff(
|
||||
self,
|
||||
from_id: Optional[str] = None,
|
||||
@@ -467,7 +560,8 @@ class CheckpointService:
|
||||
id=parts[0],
|
||||
short_id=parts[0][:8],
|
||||
message=parts[1],
|
||||
timestamp=_parse_iso_timestamp(parts[2])
|
||||
timestamp=_parse_iso_timestamp(parts[2]),
|
||||
step=_extract_step(parts[1])
|
||||
))
|
||||
|
||||
return checkpoints
|
||||
@@ -504,6 +598,14 @@ class CheckpointService:
|
||||
return cp
|
||||
return None
|
||||
|
||||
async def get_checkpoint_by_step(self, step: int) -> Optional[Checkpoint]:
|
||||
"""Get the most recent checkpoint tagged with the given step index."""
|
||||
checkpoints = await self.list_checkpoints(limit=self.config.max_checkpoints)
|
||||
for cp in checkpoints: # newest-first, so returns the latest match
|
||||
if cp.step == step:
|
||||
return cp
|
||||
return None
|
||||
|
||||
async def cleanup(self):
|
||||
"""Clean up the checkpoint service."""
|
||||
# Nothing to clean up currently
|
||||
|
||||
@@ -81,6 +81,7 @@ class Checkpoint:
|
||||
files_changed: int = 0
|
||||
insertions: int = 0
|
||||
deletions: int = 0
|
||||
step: Optional[int] = None # Per-step checkpoint index, if any
|
||||
|
||||
@classmethod
|
||||
def from_git_commit(cls, commit_hash: str, message: str, timestamp: str) -> "Checkpoint":
|
||||
@@ -101,7 +102,8 @@ class Checkpoint:
|
||||
"timestamp": self.timestamp.isoformat(),
|
||||
"files_changed": self.files_changed,
|
||||
"insertions": self.insertions,
|
||||
"deletions": self.deletions
|
||||
"deletions": self.deletions,
|
||||
"step": self.step
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -305,7 +305,8 @@ class ContextCompactor:
|
||||
messages_kept=len(compacted),
|
||||
strategy_used=self.strategy,
|
||||
tool_results_pruned=tool_results_pruned,
|
||||
previous_summary_reused=getattr(self, '_used_previous_summary', False)
|
||||
previous_summary_reused=getattr(self, '_used_previous_summary', False),
|
||||
summary=self._extract_summary_text(compacted),
|
||||
)
|
||||
result.calculate_savings_pct()
|
||||
|
||||
@@ -397,7 +398,8 @@ class ContextCompactor:
|
||||
messages_kept=len(compacted),
|
||||
strategy_used=self.strategy,
|
||||
tool_results_pruned=tool_results_pruned,
|
||||
previous_summary_reused=getattr(self, '_used_previous_summary', False)
|
||||
previous_summary_reused=getattr(self, '_used_previous_summary', False),
|
||||
summary=self._extract_summary_text(compacted),
|
||||
)
|
||||
result.calculate_savings_pct()
|
||||
|
||||
@@ -409,7 +411,39 @@ class ContextCompactor:
|
||||
self._low_savings_streak += 1
|
||||
|
||||
return compacted, result
|
||||
|
||||
|
||||
def _extract_summary_text(self, compacted: List[Dict[str, Any]]) -> str:
|
||||
"""Surface the summary text produced by summarizing strategies.
|
||||
|
||||
``CompactionResult.summary`` was historically left at ``""``, so the
|
||||
distilled summary was only reachable as an in-list system message and
|
||||
never propagated to hooks or the durable session checkpoint. This
|
||||
returns the summary the strategy just injected (LLM or naive), so
|
||||
callers/persisters (e.g. ``_persist_compaction_checkpoint``) can make
|
||||
it durable. Returns ``""`` for non-summarizing strategies.
|
||||
|
||||
Only summaries produced by the *current* pass are surfaced: we read the
|
||||
message the strategy just injected into ``compacted``. We deliberately
|
||||
do NOT fall back to the instance-level ``_previous_summary``, because a
|
||||
reused compactor would then leak a stale LLM summary into a later
|
||||
TRUNCATE/SLIDING/PRUNE pass and persist an outdated checkpoint that
|
||||
drops intervening turns on resume (Issue #3062 review).
|
||||
"""
|
||||
# Summarizing strategies tag their injected message with ``_compacted``.
|
||||
for msg in reversed(compacted):
|
||||
if msg.get("_compacted") and isinstance(msg.get("content"), str):
|
||||
return msg["content"]
|
||||
# Naive ``_summarize`` injects an untagged system summary line.
|
||||
for msg in reversed(compacted):
|
||||
content = msg.get("content")
|
||||
if (
|
||||
msg.get("role") == "system"
|
||||
and isinstance(content, str)
|
||||
and content.startswith("[Previous conversation summary]")
|
||||
):
|
||||
return content
|
||||
return ""
|
||||
|
||||
def _prune_tool_results(self, messages: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], int]:
|
||||
"""
|
||||
Delegate tool result pruning to injected protocol implementation.
|
||||
|
||||
@@ -1438,11 +1438,22 @@ class MultiAgentMemoryConfig:
|
||||
}
|
||||
|
||||
|
||||
# Import ToolSearchConfig from tools module to avoid duplication
|
||||
def __get_tool_search_config():
|
||||
# Resolve ToolSearchConfig from tools module to avoid duplication.
|
||||
# Deferred via PEP 562 module-level __getattr__ so that `import Agent`
|
||||
# does not eagerly initialise the whole tools subsystem (see issue #3191).
|
||||
_TOOL_SEARCH_CONFIG_CACHE = None
|
||||
|
||||
|
||||
def _resolve_tool_search_config():
|
||||
# Memoize so the resolved class (including the ImportError fallback) is a
|
||||
# single stable object. Without this, the fallback path would define a new
|
||||
# dataclass per call, breaking isinstance() checks across invocations.
|
||||
global _TOOL_SEARCH_CONFIG_CACHE
|
||||
if _TOOL_SEARCH_CONFIG_CACHE is not None:
|
||||
return _TOOL_SEARCH_CONFIG_CACHE
|
||||
try:
|
||||
from ..tools.tool_search import ToolSearchConfig as _ToolSearchConfig
|
||||
return _ToolSearchConfig
|
||||
_TOOL_SEARCH_CONFIG_CACHE = _ToolSearchConfig
|
||||
except ImportError:
|
||||
# Fallback minimal config if tools module not available
|
||||
@dataclass
|
||||
@@ -1452,9 +1463,8 @@ def __get_tool_search_config():
|
||||
search_default_limit: int = 5
|
||||
max_search_limit: int = 20
|
||||
core_tools: Optional[FrozenSet[str]] = None
|
||||
return FallbackToolSearchConfig
|
||||
|
||||
ToolSearchConfig = __get_tool_search_config()
|
||||
_TOOL_SEARCH_CONFIG_CACHE = FallbackToolSearchConfig
|
||||
return _TOOL_SEARCH_CONFIG_CACHE
|
||||
|
||||
|
||||
class AutonomyLevel(str, Enum):
|
||||
@@ -1477,7 +1487,7 @@ CachingParam = Union[bool, CachingConfig]
|
||||
HooksParam = Union[List[Any], HooksConfig]
|
||||
SkillsParam = Union[List[str], SkillsConfig]
|
||||
AutonomyParam = Union[bool, Dict[str, Any], "AutonomyConfig"]
|
||||
ToolSearchParam = Union[bool, str, Dict[str, Any], ToolSearchConfig]
|
||||
ToolSearchParam = Union[bool, str, Dict[str, Any], "ToolSearchConfig"]
|
||||
ToolParam = Union[bool, ToolConfig] # bool = defaults, ToolConfig = custom
|
||||
|
||||
|
||||
@@ -1621,7 +1631,7 @@ def resolve_autonomy(value: AutonomyParam) -> Optional[AutonomyConfig]:
|
||||
return _resolve(value, AutonomyConfig)
|
||||
|
||||
|
||||
def resolve_tool_search(value: ToolSearchParam) -> Optional[ToolSearchConfig]:
|
||||
def resolve_tool_search(value: ToolSearchParam) -> Optional["ToolSearchConfig"]:
|
||||
"""
|
||||
Resolve tool_search= parameter following precedence ladder.
|
||||
|
||||
@@ -1631,6 +1641,9 @@ def resolve_tool_search(value: ToolSearchParam) -> Optional[ToolSearchConfig]:
|
||||
# Simple implementation since it's unused
|
||||
if value is None or value is False:
|
||||
return None
|
||||
# Lazy-load ToolSearchConfig only when tool search is actually configured,
|
||||
# so the tools subsystem is not imported on the `import Agent` path.
|
||||
ToolSearchConfig = _resolve_tool_search_config()
|
||||
if value is True:
|
||||
return ToolSearchConfig()
|
||||
if isinstance(value, str):
|
||||
@@ -1813,3 +1826,17 @@ __all__ = [
|
||||
"resolve_tools",
|
||||
"resolve_runtime",
|
||||
]
|
||||
|
||||
|
||||
def __getattr__(name):
|
||||
"""PEP 562 lazy attribute access.
|
||||
|
||||
``ToolSearchConfig`` is resolved (and the tools subsystem imported) only on
|
||||
first access, so ``from praisonaiagents import Agent`` does not eagerly load
|
||||
the entire tools package (issue #3191).
|
||||
"""
|
||||
if name == "ToolSearchConfig":
|
||||
cfg = _resolve_tool_search_config()
|
||||
globals()["ToolSearchConfig"] = cfg # cache for subsequent access
|
||||
return cfg
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -72,8 +72,29 @@ class ContextAggregator:
|
||||
self._sources: Dict[str, Tuple[Callable, int]] = {}
|
||||
|
||||
def _estimate_tokens(self, text: str) -> int:
|
||||
"""Estimate token count (4 chars per token heuristic)."""
|
||||
return len(text) // 4 + 1
|
||||
"""Estimate token count via the canonical heuristic."""
|
||||
from .tokens import estimate_tokens_heuristic
|
||||
return estimate_tokens_heuristic(text)
|
||||
|
||||
def _truncate_to_tokens(self, text: str, max_tokens: int) -> str:
|
||||
"""Return the longest prefix of ``text`` whose estimated tokens fit ``max_tokens``.
|
||||
|
||||
Uses the canonical estimator so dense non-ASCII scripts (e.g. CJK) are
|
||||
truncated correctly rather than assuming a flat 4-chars-per-token ratio.
|
||||
"""
|
||||
if max_tokens <= 0 or not text:
|
||||
return ""
|
||||
if self._estimate_tokens(text) <= max_tokens:
|
||||
return text
|
||||
# Binary search for the longest prefix within budget.
|
||||
lo, hi = 0, len(text)
|
||||
while lo < hi:
|
||||
mid = (lo + hi + 1) // 2
|
||||
if self._estimate_tokens(text[:mid]) <= max_tokens:
|
||||
lo = mid
|
||||
else:
|
||||
hi = mid - 1
|
||||
return text[:lo]
|
||||
|
||||
def register_source(
|
||||
self,
|
||||
@@ -219,9 +240,15 @@ class ContextAggregator:
|
||||
# Try to fit partial
|
||||
remaining = max_tokens - total_tokens - separator_tokens
|
||||
if remaining > 100:
|
||||
# Truncate context
|
||||
chars_to_keep = int(remaining * 4 * 0.9)
|
||||
truncated = context[:chars_to_keep] + "..."
|
||||
# Reserve budget for the label and ellipsis so the retained
|
||||
# prefix (canonically estimated) stays within ``remaining``.
|
||||
ellipsis_tokens = self._estimate_tokens("...")
|
||||
label_tokens = (
|
||||
self._estimate_tokens(f"**{name}**:\n")
|
||||
if self.include_source_labels else 0
|
||||
)
|
||||
prefix_budget = remaining - ellipsis_tokens - label_tokens
|
||||
truncated = self._truncate_to_tokens(context, prefix_budget) + "..."
|
||||
|
||||
if self.include_source_labels:
|
||||
merged_parts.append(f"**{name}**:\n{truncated}")
|
||||
|
||||
@@ -166,6 +166,7 @@ class LoopGuard:
|
||||
self._turn_start_time: Optional[float] = None
|
||||
self._tool_counts: Dict[str, int] = {}
|
||||
self._last_progress_count = 0
|
||||
self._last_progress_action_index = 0
|
||||
|
||||
def reset_turn(self) -> None:
|
||||
"""Reset tracking for a new chat turn."""
|
||||
@@ -174,9 +175,23 @@ class LoopGuard:
|
||||
# Reset the underlying DoomLoopDetector to clear cross-turn state
|
||||
self.detector.start_session()
|
||||
self._last_progress_count = 0
|
||||
self._last_progress_action_index = 0
|
||||
|
||||
def record(self, tool_name: str, args: Dict[str, Any], success: bool) -> None:
|
||||
"""Record a tool execution for loop detection."""
|
||||
def record(
|
||||
self,
|
||||
tool_name: str,
|
||||
args: Dict[str, Any],
|
||||
success: bool,
|
||||
result: Any = None,
|
||||
) -> None:
|
||||
"""Record a tool execution for loop detection.
|
||||
|
||||
The actual tool ``result`` is used (when provided) to fingerprint the
|
||||
output so that legitimate progress is recognised. Async status-polling
|
||||
tools that return changing output (e.g. ``IN_PROGRESS`` -> ``COMPLETE``)
|
||||
are therefore not flagged as "no progress"; only genuinely identical
|
||||
repeated results are.
|
||||
"""
|
||||
if not self.config.enabled:
|
||||
return
|
||||
|
||||
@@ -184,11 +199,22 @@ class LoopGuard:
|
||||
if self._turn_start_time is None:
|
||||
self.reset_turn()
|
||||
|
||||
# Record in underlying detector
|
||||
# Record in underlying detector. Prefer the real tool result for
|
||||
# fingerprinting; fall back to the success flag when unavailable.
|
||||
# The upstream detector drops the fingerprint for falsy results
|
||||
# (``if result else None``), so wrap falsy primitives in their ``repr``
|
||||
# to keep them fingerprintable — otherwise tools that repeatedly return
|
||||
# ``""`` / ``[]`` / ``{}`` / ``0`` / ``False`` would silently bypass the
|
||||
# no-progress guard.
|
||||
safe_result = result if result is not None else success
|
||||
if not safe_result and isinstance(
|
||||
safe_result, (str, list, dict, tuple, set, bool, int, float)
|
||||
):
|
||||
safe_result = f"__loopguard_falsy__:{safe_result!r}"
|
||||
self.detector.record_action(
|
||||
action_type=tool_name,
|
||||
args=args,
|
||||
result=success, # Simple success flag for now
|
||||
result=safe_result,
|
||||
success=success,
|
||||
)
|
||||
|
||||
@@ -239,12 +265,34 @@ class LoopGuard:
|
||||
no_progress_decision = self._check_no_progress()
|
||||
if no_progress_decision and no_progress_decision.action != GuardAction.ALLOW:
|
||||
return no_progress_decision
|
||||
|
||||
|
||||
# Count-based thresholds are only meaningful as a *loop* signal when the
|
||||
# tool is actually stuck (repeating identical results). A tool that is
|
||||
# called many times but keeps producing changing output (distinct args
|
||||
# or async polling that transitions state) is making progress and must
|
||||
# not be blocked/halted purely on frequency — that is the #3073 class of
|
||||
# false positive. So block/halt only when there is a genuine
|
||||
# identical-result streak; otherwise cap the count-based verdict at WARN.
|
||||
stuck_run = self._trailing_identical_result_run()
|
||||
|
||||
# Classify tool and apply appropriate thresholds
|
||||
if self._is_idempotent_tool(tool_name):
|
||||
return self._check_idempotent_tool(tool_name, current_count, args)
|
||||
decision = self._check_idempotent_tool(tool_name, current_count, args)
|
||||
else:
|
||||
return self._check_mutating_tool(tool_name, current_count, args)
|
||||
decision = self._check_mutating_tool(tool_name, current_count, args)
|
||||
|
||||
if decision.should_block() and stuck_run < self.config.no_progress_warn:
|
||||
return LoopGuardDecision(
|
||||
action=GuardAction.WARN,
|
||||
code=decision.code.replace("_halt", "_warn").replace("_block", "_warn"),
|
||||
message=(
|
||||
f"Tool '{tool_name}' called {current_count} times this turn. "
|
||||
"Results are still changing (progress detected); continuing, "
|
||||
"but consider whether this many calls are necessary."
|
||||
),
|
||||
metadata=decision.metadata,
|
||||
)
|
||||
return decision
|
||||
|
||||
def _is_idempotent_tool(self, tool_name: str) -> bool:
|
||||
"""Check if a tool is classified as idempotent."""
|
||||
@@ -331,30 +379,74 @@ class LoopGuard:
|
||||
)
|
||||
|
||||
def _check_no_progress(self) -> Optional[LoopGuardDecision]:
|
||||
"""Check for no-progress patterns."""
|
||||
current_progress = len(self.detector._progress_markers)
|
||||
actions_since_progress = len(self.detector._actions) - self._last_progress_count
|
||||
|
||||
if actions_since_progress >= self.config.no_progress_halt:
|
||||
"""Check for no-progress patterns.
|
||||
|
||||
"No progress" means the agent keeps producing the *same* tool results
|
||||
without moving forward. It is measured by the length of the trailing run
|
||||
of consecutive identical tool results, not by the raw tool-call count.
|
||||
This ensures legitimate long-running / async workflows (e.g. polling a
|
||||
job status that transitions ``IN_PROGRESS`` -> ``COMPLETE``, or pacing
|
||||
with a ``wait`` tool) are not penalised: any change in results, or an
|
||||
explicit progress marker, resets the streak.
|
||||
"""
|
||||
stuck_run = self._trailing_identical_result_run()
|
||||
|
||||
if stuck_run >= self.config.no_progress_halt:
|
||||
return LoopGuardDecision(
|
||||
action=GuardAction.HALT,
|
||||
code="no_progress_halt",
|
||||
message=f"No progress detected in {actions_since_progress} tool calls. Agent may be stuck.",
|
||||
metadata={"actions_since_progress": actions_since_progress}
|
||||
message=f"No progress detected in {stuck_run} tool calls. Agent may be stuck.",
|
||||
metadata={"actions_since_progress": stuck_run}
|
||||
)
|
||||
elif actions_since_progress >= self.config.no_progress_warn:
|
||||
elif stuck_run >= self.config.no_progress_warn:
|
||||
return LoopGuardDecision(
|
||||
action=GuardAction.WARN,
|
||||
code="no_progress_warn",
|
||||
message=f"Limited progress in {actions_since_progress} tool calls. Consider changing approach.",
|
||||
metadata={"actions_since_progress": actions_since_progress}
|
||||
message=f"Limited progress in {stuck_run} tool calls. Consider changing approach.",
|
||||
metadata={"actions_since_progress": stuck_run}
|
||||
)
|
||||
return None
|
||||
|
||||
def _trailing_identical_result_run(self) -> int:
|
||||
"""Count trailing tool calls that produced the same result fingerprint.
|
||||
|
||||
A change in ``result_hash`` (distinct tool output) breaks the streak and
|
||||
is treated as progress. The streak also requires the *same tool* so that
|
||||
unrelated tools returning a common value (e.g. ``"ok"`` / ``True``) are
|
||||
not lumped into one stuck run. Actions without a result fingerprint do
|
||||
not extend a stuck streak, and any action recorded before the most
|
||||
recent explicit progress marker is excluded from the run.
|
||||
"""
|
||||
actions = self.detector._actions
|
||||
limit = self._last_progress_action_index
|
||||
if not actions or len(actions) <= limit:
|
||||
return 0
|
||||
|
||||
last = actions[-1]
|
||||
if last.result_hash is None:
|
||||
return 0
|
||||
|
||||
run = 0
|
||||
for idx in range(len(actions) - 1, limit - 1, -1):
|
||||
action = actions[idx]
|
||||
if (
|
||||
action.result_hash == last.result_hash
|
||||
and action.action_type == last.action_type
|
||||
):
|
||||
run += 1
|
||||
else:
|
||||
break
|
||||
return run
|
||||
|
||||
def mark_progress(self, marker: str) -> None:
|
||||
"""Mark that meaningful progress has been made."""
|
||||
"""Mark that meaningful progress has been made.
|
||||
|
||||
Records the current action index as a boundary so that only tool calls
|
||||
made *after* this marker can contribute to a future no-progress streak.
|
||||
"""
|
||||
self.detector.mark_progress(marker)
|
||||
self._last_progress_count = len(self.detector._progress_markers)
|
||||
self._last_progress_action_index = len(self.detector._actions)
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""Get loop guard statistics."""
|
||||
|
||||
@@ -104,6 +104,9 @@ __all__ = [
|
||||
# LoopEvaluator (loop health: convergence, waste, doom-loop guards)
|
||||
"LoopEvaluator",
|
||||
"LoopHealthResult",
|
||||
# PromptOptimizer (optimise agent.instructions against an eval, keep best)
|
||||
"PromptOptimizer",
|
||||
"OptimizeResult",
|
||||
]
|
||||
|
||||
_LAZY_IMPORTS = {
|
||||
@@ -185,6 +188,9 @@ _LAZY_IMPORTS = {
|
||||
# LoopEvaluator (loop health: convergence, waste, doom-loop guards)
|
||||
"LoopEvaluator": ("loop_eval", "LoopEvaluator"),
|
||||
"LoopHealthResult": ("loop_eval", "LoopHealthResult"),
|
||||
# PromptOptimizer (optimise agent.instructions against an eval, keep best)
|
||||
"PromptOptimizer": ("prompt_optimizer", "PromptOptimizer"),
|
||||
"OptimizeResult": ("prompt_optimizer", "OptimizeResult"),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ Example:
|
||||
"""
|
||||
|
||||
import time
|
||||
import math
|
||||
import logging
|
||||
from praisonaiagents._logging import get_logger
|
||||
from typing import TYPE_CHECKING, Optional, Callable, Any, List
|
||||
@@ -27,6 +28,17 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@dataclass
|
||||
class _MetricScore:
|
||||
"""Minimal score result for the numeric-metric path (Judge-shaped)."""
|
||||
score: float
|
||||
reasoning: str = "numeric metric"
|
||||
suggestions: List[str] = None # type: ignore[assignment]
|
||||
|
||||
def __post_init__(self):
|
||||
if self.suggestions is None:
|
||||
self.suggestions = []
|
||||
|
||||
@dataclass
|
||||
class EvaluationLoopConfig:
|
||||
"""Configuration for EvaluationLoop."""
|
||||
@@ -36,6 +48,7 @@ class EvaluationLoopConfig:
|
||||
mode: str = "optimize"
|
||||
model: str = "gpt-4o-mini"
|
||||
verbose: bool = False
|
||||
metric: Optional[Callable[[str], float]] = None
|
||||
|
||||
class EvaluationLoop:
|
||||
"""
|
||||
@@ -79,6 +92,7 @@ class EvaluationLoop:
|
||||
on_iteration: Optional[Callable[[Any], None]] = None,
|
||||
verbose: bool = False,
|
||||
model: str = "gpt-4o-mini",
|
||||
metric: Optional[Callable[[str], float]] = None,
|
||||
):
|
||||
self.agent = agent
|
||||
self.criteria = criteria
|
||||
@@ -89,6 +103,7 @@ class EvaluationLoop:
|
||||
self.on_iteration = on_iteration
|
||||
self.verbose = verbose
|
||||
self.model = model
|
||||
self.metric = metric
|
||||
|
||||
if mode not in ("optimize", "review"):
|
||||
raise ValueError(f"mode must be 'optimize' or 'review', got '{mode}'")
|
||||
@@ -104,6 +119,37 @@ class EvaluationLoop:
|
||||
)
|
||||
return self._judge
|
||||
|
||||
def _score(self, output: str):
|
||||
"""Score an output via a numeric metric (if set) or the Judge.
|
||||
|
||||
Returns a lightweight result object exposing ``score``, ``reasoning``
|
||||
and ``suggestions`` so the loop can treat both paths uniformly.
|
||||
"""
|
||||
if self.metric is not None:
|
||||
return _MetricScore(score=self._sanitize_score(self.metric(output)))
|
||||
return self.judge.run(output=output, criteria=self.criteria)
|
||||
|
||||
@staticmethod
|
||||
def _sanitize_score(value: Any) -> float:
|
||||
"""Coerce a metric result to a finite float (NaN/inf -> 0.0).
|
||||
|
||||
Non-finite scores never reach the threshold and make ``max()`` selection
|
||||
order-dependent, so they are floored to the worst score.
|
||||
"""
|
||||
score = float(value)
|
||||
if not math.isfinite(score):
|
||||
logger.warning("Metric returned non-finite value %r; treating as 0.0", score)
|
||||
return 0.0
|
||||
return score
|
||||
|
||||
async def _score_async(self, output: str):
|
||||
"""Async twin of :meth:`_score`."""
|
||||
if self.metric is not None:
|
||||
return _MetricScore(score=self._sanitize_score(self.metric(output)))
|
||||
if hasattr(self.judge, 'run_async'):
|
||||
return await self.judge.run_async(output=output, criteria=self.criteria)
|
||||
return self.judge.run(output=output, criteria=self.criteria)
|
||||
|
||||
def _get_agent_output(self, prompt: str, iteration: int, feedback: str = "") -> str:
|
||||
"""Get output from agent, optionally including feedback from previous iteration."""
|
||||
if iteration == 1 or not feedback:
|
||||
@@ -146,10 +192,7 @@ class EvaluationLoop:
|
||||
|
||||
output = self._get_agent_output(prompt, i, feedback)
|
||||
|
||||
judge_result = self.judge.run(
|
||||
output=output,
|
||||
criteria=self.criteria,
|
||||
)
|
||||
judge_result = self._score(output)
|
||||
|
||||
findings = getattr(judge_result, 'suggestions', []) or []
|
||||
|
||||
@@ -157,7 +200,7 @@ class EvaluationLoop:
|
||||
iteration=i,
|
||||
output=output,
|
||||
score=judge_result.score,
|
||||
reasoning=judge_result.reasoning,
|
||||
reasoning=getattr(judge_result, 'reasoning', ''),
|
||||
findings=findings,
|
||||
)
|
||||
iterations.append(iteration_result)
|
||||
@@ -173,12 +216,13 @@ class EvaluationLoop:
|
||||
logger.info(f"Threshold met at iteration {i}: {judge_result.score} >= {self.threshold}")
|
||||
break
|
||||
|
||||
feedback = judge_result.reasoning
|
||||
feedback = getattr(judge_result, 'reasoning', '')
|
||||
if findings:
|
||||
feedback += "\nSuggestions:\n" + "\n".join(f"- {s}" for s in findings)
|
||||
|
||||
total_duration = time.time() - start_time
|
||||
success = iterations[-1].score >= self.threshold if iterations else False
|
||||
best = max(iterations, key=lambda it: it.score) if iterations else None
|
||||
success = best.score >= self.threshold if best else False
|
||||
|
||||
result = EvaluationLoopResult(
|
||||
iterations=iterations,
|
||||
@@ -186,6 +230,7 @@ class EvaluationLoop:
|
||||
total_duration_seconds=total_duration,
|
||||
threshold=self.threshold,
|
||||
mode=self.mode,
|
||||
best=best,
|
||||
)
|
||||
|
||||
if self.verbose:
|
||||
@@ -215,16 +260,7 @@ class EvaluationLoop:
|
||||
|
||||
output = await self._get_agent_output_async(prompt, i, feedback)
|
||||
|
||||
if hasattr(self.judge, 'run_async'):
|
||||
judge_result = await self.judge.run_async(
|
||||
output=output,
|
||||
criteria=self.criteria,
|
||||
)
|
||||
else:
|
||||
judge_result = self.judge.run(
|
||||
output=output,
|
||||
criteria=self.criteria,
|
||||
)
|
||||
judge_result = await self._score_async(output)
|
||||
|
||||
findings = getattr(judge_result, 'suggestions', []) or []
|
||||
|
||||
@@ -232,7 +268,7 @@ class EvaluationLoop:
|
||||
iteration=i,
|
||||
output=output,
|
||||
score=judge_result.score,
|
||||
reasoning=judge_result.reasoning,
|
||||
reasoning=getattr(judge_result, 'reasoning', ''),
|
||||
findings=findings,
|
||||
)
|
||||
iterations.append(iteration_result)
|
||||
@@ -248,12 +284,13 @@ class EvaluationLoop:
|
||||
logger.info(f"Threshold met at iteration {i}: {judge_result.score} >= {self.threshold}")
|
||||
break
|
||||
|
||||
feedback = judge_result.reasoning
|
||||
feedback = getattr(judge_result, 'reasoning', '')
|
||||
if findings:
|
||||
feedback += "\nSuggestions:\n" + "\n".join(f"- {s}" for s in findings)
|
||||
|
||||
total_duration = time.time() - start_time
|
||||
success = iterations[-1].score >= self.threshold if iterations else False
|
||||
best = max(iterations, key=lambda it: it.score) if iterations else None
|
||||
success = best.score >= self.threshold if best else False
|
||||
|
||||
result = EvaluationLoopResult(
|
||||
iterations=iterations,
|
||||
@@ -261,6 +298,7 @@ class EvaluationLoop:
|
||||
total_duration_seconds=total_duration,
|
||||
threshold=self.threshold,
|
||||
mode=self.mode,
|
||||
best=best,
|
||||
)
|
||||
|
||||
if self.verbose:
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
"""
|
||||
PromptOptimizer - automatic optimisation of an agent's instructions against an eval.
|
||||
|
||||
Given an agent, a small eval set, and a scorer (LLM ``Judge`` by default or a
|
||||
user-supplied numeric metric), the optimiser generates N instruction candidates,
|
||||
evaluates each candidate over the eval set, keeps the highest-scoring one, and
|
||||
(optionally) writes the winning instructions back to ``agent.instructions``.
|
||||
|
||||
This is the bounded "keep-the-best prompt" slice — it does NOT do tree/beam
|
||||
search over code variants or an agent-rewrites-its-own-harness loop.
|
||||
|
||||
Example:
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.eval import PromptOptimizer
|
||||
|
||||
agent = Agent(name="summariser", instructions="Summarise the input.")
|
||||
result = PromptOptimizer(
|
||||
agent=agent,
|
||||
evalset=[("summarise X", gold_x), ("summarise Y", gold_y)],
|
||||
metric=rouge_l, # numeric metric, or omit to use the LLM Judge
|
||||
n_candidates=6,
|
||||
).optimize()
|
||||
print(result.best_score, result.best_instructions)
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import math
|
||||
from praisonaiagents._logging import get_logger
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, Callable, Generator, List, Optional, Tuple
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..agent.agent import Agent
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# An eval case is (prompt, expected). ``expected`` may be None when using a
|
||||
# Judge with only criteria.
|
||||
EvalCase = Tuple[str, Any]
|
||||
|
||||
|
||||
@dataclass
|
||||
class OptimizeResult:
|
||||
"""Result of a prompt-optimisation run.
|
||||
|
||||
Attributes:
|
||||
best_instructions: The highest-scoring instructions found.
|
||||
best_score: The aggregate score of the winning instructions.
|
||||
base_score: The aggregate score of the original instructions.
|
||||
trials: List of ``(instructions, score)`` for every candidate tried,
|
||||
including the base instructions.
|
||||
applied: Whether the winning instructions were written back to the agent.
|
||||
"""
|
||||
best_instructions: str
|
||||
best_score: float
|
||||
base_score: float = 0.0
|
||||
trials: List[Tuple[str, float]] = field(default_factory=list)
|
||||
applied: bool = False
|
||||
|
||||
@property
|
||||
def improved(self) -> bool:
|
||||
"""True when the best candidate scored strictly higher than the base."""
|
||||
return self.best_score > self.base_score
|
||||
|
||||
|
||||
class PromptOptimizer:
|
||||
"""Optimise an agent's instructions against an eval set, keeping the best.
|
||||
|
||||
Args:
|
||||
agent: The Agent whose ``instructions`` will be optimised.
|
||||
evalset: List of ``(prompt, expected)`` cases to score candidates on.
|
||||
scorer: Optional custom ``Judge`` instance (defaults to a new ``Judge``).
|
||||
Ignored when ``metric`` is provided.
|
||||
metric: Optional numeric metric ``(output, expected) -> float``. When
|
||||
set, empirical scoring replaces the LLM Judge.
|
||||
criteria: Optional criteria passed to the default Judge.
|
||||
n_candidates: Number of instruction candidates to generate (default: 6).
|
||||
model: LLM model used to propose candidates / judge (default: gpt-4o-mini).
|
||||
apply: Write the winning instructions back to the agent (default: True).
|
||||
"""
|
||||
|
||||
_DELIMITER = "==="
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
agent: "Agent",
|
||||
evalset: List[EvalCase],
|
||||
*,
|
||||
scorer: Optional[Any] = None,
|
||||
metric: Optional[Callable[[str, Any], float]] = None,
|
||||
criteria: str = "",
|
||||
n_candidates: int = 6,
|
||||
model: str = "gpt-4o-mini",
|
||||
apply: bool = True,
|
||||
):
|
||||
if not evalset:
|
||||
raise ValueError("evalset must contain at least one (prompt, expected) case")
|
||||
self.agent = agent
|
||||
self.evalset = list(evalset)
|
||||
self._scorer = scorer
|
||||
self.metric = metric
|
||||
self.criteria = criteria
|
||||
self.n_candidates = n_candidates
|
||||
self.model = model
|
||||
self.apply = apply
|
||||
self.trials: List[Tuple[str, float]] = []
|
||||
|
||||
@property
|
||||
def scorer(self):
|
||||
"""Lazy-load a Judge for scoring (only when no numeric metric is set)."""
|
||||
if self.metric is not None:
|
||||
return None
|
||||
if self._scorer is None:
|
||||
from .judge import Judge
|
||||
self._scorer = Judge(criteria=self.criteria, model=self.model)
|
||||
return self._scorer
|
||||
|
||||
def _score_one(self, output: str, expected: Any) -> float:
|
||||
"""Score a single output against its expected value."""
|
||||
if self.metric is not None:
|
||||
value = float(self.metric(output, expected))
|
||||
else:
|
||||
result = self.scorer.run(output=output, expected=expected, criteria=self.criteria)
|
||||
value = float(result.score)
|
||||
# Non-finite scores (NaN/inf) break ``max()`` and threshold logic; treat
|
||||
# them as the worst possible score so they never win selection.
|
||||
if not math.isfinite(value):
|
||||
logger.warning("Scorer returned non-finite value %r; treating as 0.0", value)
|
||||
return 0.0
|
||||
return value
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _applied(self, instructions: str) -> Generator[None, None, None]:
|
||||
"""Temporarily make ``instructions`` the agent's *effective* prompt.
|
||||
|
||||
The agent's system prompt is derived from ``goal``/``backstory`` (seeded
|
||||
from ``instructions`` at construction) and is cached; swapping only
|
||||
``instructions`` would leave chat behaviour unchanged. This swaps all
|
||||
three fields, clears the system-prompt cache so the change takes effect,
|
||||
and isolates chat history so eval turns never pollute the live agent.
|
||||
Everything is restored on exit, even on error.
|
||||
"""
|
||||
agent = self.agent
|
||||
original = (
|
||||
getattr(agent, "instructions", None),
|
||||
getattr(agent, "goal", None),
|
||||
getattr(agent, "backstory", None),
|
||||
)
|
||||
cache = getattr(agent, "_system_prompt_cache", None)
|
||||
try:
|
||||
agent.instructions = instructions
|
||||
if hasattr(agent, "goal"):
|
||||
agent.goal = instructions
|
||||
if hasattr(agent, "backstory"):
|
||||
agent.backstory = instructions
|
||||
if cache is not None:
|
||||
cache.clear()
|
||||
ephemeral = getattr(agent, "ephemeral", None)
|
||||
if callable(ephemeral):
|
||||
with ephemeral():
|
||||
yield
|
||||
else:
|
||||
yield
|
||||
finally:
|
||||
agent.instructions, goal, backstory = original[0], original[1], original[2]
|
||||
if goal is not None and hasattr(agent, "goal"):
|
||||
agent.goal = goal
|
||||
if backstory is not None and hasattr(agent, "backstory"):
|
||||
agent.backstory = backstory
|
||||
if cache is not None:
|
||||
cache.clear()
|
||||
|
||||
def _score_instructions(self, instructions: str) -> float:
|
||||
"""Run the agent (with ``instructions``) over the eval set and aggregate.
|
||||
|
||||
Temporarily makes ``instructions`` the agent's effective prompt, runs
|
||||
each prompt, scores it, and returns the mean score. Always restores the
|
||||
original agent state.
|
||||
"""
|
||||
scores: List[float] = []
|
||||
with self._applied(instructions):
|
||||
for prompt, expected in self.evalset:
|
||||
output = str(self.agent.chat(prompt))
|
||||
scores.append(self._score_one(output, expected))
|
||||
return sum(scores) / len(scores) if scores else 0.0
|
||||
|
||||
def _lowest_scoring_examples(self, instructions: str, limit: int = 2) -> List[str]:
|
||||
"""Return prompts where ``instructions`` scored worst (reflective signal)."""
|
||||
scored: List[Tuple[float, str]] = []
|
||||
with self._applied(instructions):
|
||||
for prompt, expected in self.evalset:
|
||||
output = str(self.agent.chat(prompt))
|
||||
scored.append((self._score_one(output, expected), prompt))
|
||||
scored.sort(key=lambda x: x[0])
|
||||
return [p for _, p in scored[:limit]]
|
||||
|
||||
def _propose_variants(self, base: str) -> List[str]:
|
||||
"""Ask an auxiliary LLM to rewrite ``base`` into diverse candidates."""
|
||||
weak = self._lowest_scoring_examples(base)
|
||||
weak_block = ""
|
||||
if weak:
|
||||
weak_block = "\n\nThese example prompts scored poorly; address them:\n" + \
|
||||
"\n".join(f"- {p}" for p in weak)
|
||||
proposal_prompt = (
|
||||
"You are optimising an AI agent's system instructions. "
|
||||
f"Rewrite the instructions below into {self.n_candidates} distinct, "
|
||||
"improved variants. Each variant must be a complete, standalone set of "
|
||||
"instructions (which may span multiple lines) that stays faithful to "
|
||||
"the original intent while being clearer and more effective. Separate "
|
||||
f"each variant with a line containing only {self._DELIMITER!r}. "
|
||||
"No numbering, no commentary.\n\n"
|
||||
f"Current instructions:\n{base}{weak_block}"
|
||||
)
|
||||
from ..agent.agent import Agent
|
||||
proposer = Agent(
|
||||
name="prompt-optimizer",
|
||||
instructions="You rewrite system prompts into improved variants.",
|
||||
llm=self.model,
|
||||
)
|
||||
response = str(proposer.chat(proposal_prompt) or "")
|
||||
variants = self._split_variants(response)
|
||||
seen = set()
|
||||
unique: List[str] = []
|
||||
for v in variants:
|
||||
if v and v != base and v not in seen:
|
||||
seen.add(v)
|
||||
unique.append(v)
|
||||
return unique[: self.n_candidates]
|
||||
|
||||
def _split_variants(self, response: str) -> List[str]:
|
||||
"""Parse the proposer response into complete (possibly multiline) variants.
|
||||
|
||||
Splits on the explicit delimiter when present, otherwise falls back to
|
||||
blank-line-separated blocks, so a multiline prompt is kept intact rather
|
||||
than treated as many single-line fragments.
|
||||
"""
|
||||
text = response.strip()
|
||||
if not text:
|
||||
return []
|
||||
if self._DELIMITER in text:
|
||||
blocks = text.split(self._DELIMITER)
|
||||
else:
|
||||
import re
|
||||
blocks = re.split(r"\n\s*\n", text)
|
||||
return [b.strip().strip("-").strip() for b in blocks if b.strip()]
|
||||
|
||||
def _apply_permanently(self, instructions: str) -> None:
|
||||
"""Write winning instructions to the fields that drive the system prompt.
|
||||
|
||||
The chat system prompt is built from ``goal``/``backstory`` (both seeded
|
||||
from ``instructions`` at construction) and cached; writing all three and
|
||||
clearing the cache ensures the applied instructions actually take effect.
|
||||
"""
|
||||
agent = self.agent
|
||||
agent.instructions = instructions
|
||||
if hasattr(agent, "goal"):
|
||||
agent.goal = instructions
|
||||
if hasattr(agent, "backstory"):
|
||||
agent.backstory = instructions
|
||||
cache = getattr(agent, "_system_prompt_cache", None)
|
||||
if cache is not None:
|
||||
cache.clear()
|
||||
|
||||
def optimize(self) -> OptimizeResult:
|
||||
"""Generate candidates, keep the best, and (optionally) apply it."""
|
||||
base = self.agent.instructions or ""
|
||||
base_score = self._score_instructions(base)
|
||||
self.trials = [(base, base_score)]
|
||||
best_instructions, best_score = base, base_score
|
||||
|
||||
for candidate in self._propose_variants(base):
|
||||
score = self._score_instructions(candidate)
|
||||
self.trials.append((candidate, score))
|
||||
if score > best_score:
|
||||
best_instructions, best_score = candidate, score
|
||||
|
||||
applied = False
|
||||
if self.apply and best_instructions != base:
|
||||
self._apply_permanently(best_instructions)
|
||||
applied = True
|
||||
|
||||
return OptimizeResult(
|
||||
best_instructions=best_instructions,
|
||||
best_score=best_score,
|
||||
base_score=base_score,
|
||||
trials=list(self.trials),
|
||||
applied=applied,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"PromptOptimizer",
|
||||
"OptimizeResult",
|
||||
]
|
||||
@@ -695,6 +695,28 @@ class EvaluationLoopResult:
|
||||
threshold: float = 8.0
|
||||
mode: str = "optimize"
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
best: Optional[IterationResult] = None
|
||||
|
||||
@property
|
||||
def best_iteration(self) -> Optional[IterationResult]:
|
||||
"""Get the highest-scoring iteration (keep-the-best, not the last)."""
|
||||
if self.best is not None:
|
||||
return self.best
|
||||
if not self.iterations:
|
||||
return None
|
||||
return max(self.iterations, key=lambda it: it.score)
|
||||
|
||||
@property
|
||||
def best_score(self) -> float:
|
||||
"""Get the highest score achieved across iterations."""
|
||||
best = self.best_iteration
|
||||
return best.score if best is not None else 0.0
|
||||
|
||||
@property
|
||||
def best_output(self) -> str:
|
||||
"""Get the output of the highest-scoring iteration."""
|
||||
best = self.best_iteration
|
||||
return best.output if best is not None else ""
|
||||
|
||||
@property
|
||||
def final_score(self) -> float:
|
||||
@@ -733,6 +755,8 @@ class EvaluationLoopResult:
|
||||
return {
|
||||
"success": self.success,
|
||||
"final_score": self.final_score,
|
||||
"best_score": self.best_score,
|
||||
"best_output": self.best_output,
|
||||
"score_history": self.score_history,
|
||||
"final_output": self.final_output,
|
||||
"accumulated_findings": self.accumulated_findings,
|
||||
|
||||
@@ -22,26 +22,22 @@ import logging
|
||||
from praisonaiagents._logging import get_logger
|
||||
from typing import Dict, Optional, Union
|
||||
|
||||
from ..context.budgeter import MODEL_LIMITS as _CANONICAL_LIMITS
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Default context lengths for common models (fallback when litellm unavailable)
|
||||
# Based on official documentation as of January 2025
|
||||
DEFAULT_CONTEXT_LENGTHS: Dict[str, int] = {
|
||||
# Eval-specific context lengths not present in the canonical budgeter table
|
||||
# (dated Anthropic snapshots, o1 variants, Mistral/DeepSeek/Groq, etc.).
|
||||
# Based on official documentation as of January 2025.
|
||||
_EVAL_EXTRA_LENGTHS: Dict[str, int] = {
|
||||
# OpenAI models
|
||||
"gpt-4o": 128000,
|
||||
"gpt-4o-mini": 128000,
|
||||
"gpt-4-turbo": 128000,
|
||||
"gpt-4-turbo-preview": 128000,
|
||||
"gpt-4": 8192,
|
||||
"gpt-4-32k": 32768,
|
||||
"gpt-3.5-turbo": 16385,
|
||||
"gpt-3.5-turbo-16k": 16385,
|
||||
"o1": 200000,
|
||||
"o1-mini": 128000,
|
||||
"o1-preview": 128000,
|
||||
"o3-mini": 200000,
|
||||
|
||||
# Anthropic models
|
||||
|
||||
# Anthropic models (dated snapshots)
|
||||
"claude-3-5-sonnet-20241022": 200000,
|
||||
"claude-3-5-sonnet-latest": 200000,
|
||||
"claude-3-5-haiku-20241022": 200000,
|
||||
@@ -50,24 +46,22 @@ DEFAULT_CONTEXT_LENGTHS: Dict[str, int] = {
|
||||
"claude-3-haiku-20240307": 200000,
|
||||
"claude-2.1": 200000,
|
||||
"claude-2": 100000,
|
||||
|
||||
|
||||
# Google models
|
||||
"gemini-1.5-pro": 2097152,
|
||||
"gemini-1.5-flash": 1048576,
|
||||
"gemini-1.5-flash-8b": 1048576,
|
||||
"gemini-2.0-flash-exp": 1048576,
|
||||
"gemini-pro": 32760,
|
||||
|
||||
|
||||
# Mistral models
|
||||
"mistral-large-latest": 128000,
|
||||
"mistral-medium-latest": 32000,
|
||||
"mistral-small-latest": 32000,
|
||||
"codestral-latest": 32000,
|
||||
|
||||
|
||||
# DeepSeek models
|
||||
"deepseek-chat": 64000,
|
||||
"deepseek-coder": 64000,
|
||||
|
||||
|
||||
# Groq models
|
||||
"llama-3.3-70b-versatile": 128000,
|
||||
"llama-3.1-70b-versatile": 128000,
|
||||
@@ -75,6 +69,36 @@ DEFAULT_CONTEXT_LENGTHS: Dict[str, int] = {
|
||||
"mixtral-8x7b-32768": 32768,
|
||||
}
|
||||
|
||||
# Guard the invariant that eval-only extras never silently shadow canonical
|
||||
# entries. If a key is later added to the canonical budgeter table, it should be
|
||||
# removed from _EVAL_EXTRA_LENGTHS so the single source of truth stays authoritative.
|
||||
_overlapping_keys = set(_EVAL_EXTRA_LENGTHS) & {
|
||||
k for k in _CANONICAL_LIMITS if k != "default"
|
||||
}
|
||||
assert not _overlapping_keys, (
|
||||
"_EVAL_EXTRA_LENGTHS keys overlap with canonical MODEL_LIMITS; remove the "
|
||||
f"duplicate(s) from the eval table: {sorted(_overlapping_keys)}"
|
||||
)
|
||||
|
||||
# Default context lengths for common models (fallback when litellm unavailable).
|
||||
# Built by merging the canonical budgeter table (context/budgeter.MODEL_LIMITS)
|
||||
# with eval-specific extras, so a single source of truth is maintained and new
|
||||
# model families (e.g. gpt-5, gpt-4.1) resolve consistently across both paths.
|
||||
#
|
||||
# Keys are ordered by descending length so that get_context_length's partial
|
||||
# matching checks specific names (e.g. "gpt-4-32k") before shorter prefixes
|
||||
# (e.g. "gpt-4") and returns the most precise context window.
|
||||
DEFAULT_CONTEXT_LENGTHS: Dict[str, int] = dict(
|
||||
sorted(
|
||||
{
|
||||
**{k: v for k, v in _CANONICAL_LIMITS.items() if k != "default"},
|
||||
**_EVAL_EXTRA_LENGTHS,
|
||||
}.items(),
|
||||
key=lambda item: len(item[0]),
|
||||
reverse=True,
|
||||
)
|
||||
)
|
||||
|
||||
# Default context length for unknown models
|
||||
DEFAULT_CONTEXT_LENGTH = 128000
|
||||
|
||||
|
||||
@@ -21,6 +21,11 @@ from .protocols import (
|
||||
EventType,
|
||||
OperatorScope,
|
||||
GatewayCloseCode,
|
||||
# Declarative method -> required-scope registry (Issue #3206)
|
||||
GatewayMethodDescriptor,
|
||||
GATEWAY_METHODS,
|
||||
register_gateway_method,
|
||||
resolve_required_scope,
|
||||
# Config hot-reload observability (Issue #3049)
|
||||
ReloadStatus,
|
||||
compute_config_revision,
|
||||
@@ -86,6 +91,7 @@ from .protocols import (
|
||||
GATEWAY_FATAL_CONFIG_EXIT_CODE,
|
||||
FatalConfigError,
|
||||
classify_exit_reason,
|
||||
RestartLoopGuard,
|
||||
# Protocol version negotiation
|
||||
PROTOCOL_VERSION,
|
||||
MIN_PROTOCOL_VERSION,
|
||||
@@ -125,6 +131,11 @@ from .protocols import (
|
||||
FrameDecodeError,
|
||||
ClientFrame,
|
||||
decode_client_frame,
|
||||
# Weak / placeholder secret guard (Issue #3259)
|
||||
KNOWN_WEAK_SECRETS,
|
||||
WeakGatewaySecretError,
|
||||
is_weak_secret,
|
||||
assert_gateway_secret_strong,
|
||||
)
|
||||
from .hooks import (
|
||||
HookAction,
|
||||
@@ -132,6 +143,7 @@ from .hooks import (
|
||||
InboundTriggerProtocol,
|
||||
render_template,
|
||||
compute_idempotency_key,
|
||||
verify_webhook_signature,
|
||||
)
|
||||
from .config import (
|
||||
GatewayConfig,
|
||||
@@ -208,6 +220,11 @@ __all__ = [
|
||||
"EventType",
|
||||
"OperatorScope",
|
||||
"GatewayCloseCode",
|
||||
# Declarative method -> required-scope registry (Issue #3206)
|
||||
"GatewayMethodDescriptor",
|
||||
"GATEWAY_METHODS",
|
||||
"register_gateway_method",
|
||||
"resolve_required_scope",
|
||||
# Config hot-reload observability (Issue #3049)
|
||||
"ReloadStatus",
|
||||
"compute_config_revision",
|
||||
@@ -272,6 +289,7 @@ __all__ = [
|
||||
"GATEWAY_FATAL_CONFIG_EXIT_CODE",
|
||||
"FatalConfigError",
|
||||
"classify_exit_reason",
|
||||
"RestartLoopGuard",
|
||||
# Protocol version negotiation
|
||||
"PROTOCOL_VERSION",
|
||||
"MIN_PROTOCOL_VERSION",
|
||||
@@ -311,12 +329,18 @@ __all__ = [
|
||||
"FrameDecodeError",
|
||||
"ClientFrame",
|
||||
"decode_client_frame",
|
||||
# Weak / placeholder secret guard (Issue #3259)
|
||||
"KNOWN_WEAK_SECRETS",
|
||||
"WeakGatewaySecretError",
|
||||
"is_weak_secret",
|
||||
"assert_gateway_secret_strong",
|
||||
# Inbound trigger / webhook contract (Issue #2281)
|
||||
"HookAction",
|
||||
"HookConfig",
|
||||
"InboundTriggerProtocol",
|
||||
"render_template",
|
||||
"compute_idempotency_key",
|
||||
"verify_webhook_signature",
|
||||
# Config (always available)
|
||||
"GatewayConfig",
|
||||
"SessionConfig",
|
||||
|
||||
@@ -27,9 +27,10 @@ process; this fires from an external HTTP request.
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, Optional, Protocol, runtime_checkable
|
||||
from typing import Any, Dict, List, Optional, Protocol, runtime_checkable
|
||||
|
||||
__all__ = [
|
||||
"HookAction",
|
||||
@@ -37,6 +38,7 @@ __all__ = [
|
||||
"InboundTriggerProtocol",
|
||||
"compute_idempotency_key",
|
||||
"render_template",
|
||||
"verify_webhook_signature",
|
||||
]
|
||||
|
||||
|
||||
@@ -148,6 +150,56 @@ def compute_idempotency_key(
|
||||
return hashlib.sha256(basis.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def verify_webhook_signature(
|
||||
secret: Optional[str],
|
||||
raw_body: bytes,
|
||||
signature: Optional[str],
|
||||
*,
|
||||
algo: str = "sha256",
|
||||
prefix: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""Constant-time HMAC verification of a webhook body (dependency-free).
|
||||
|
||||
A pure counterpart to ``compute_idempotency_key`` that lets the core
|
||||
contract validate a provider signature over the *raw* request bytes without
|
||||
importing the wrapper's crypto. Mirrors the wrapper's ``verify_hmac``: it is
|
||||
fail-closed (a missing secret/signature or unknown ``algo`` returns
|
||||
``False`` rather than raising) and prefix-aware.
|
||||
|
||||
Args:
|
||||
secret: Shared signing secret. Falsy → ``False``.
|
||||
raw_body: The exact raw request body bytes the provider signed.
|
||||
signature: The signature header value the provider sent.
|
||||
algo: Hash algorithm name (e.g. ``"sha256"``, ``"sha1"``).
|
||||
prefix: Optional signature prefix (e.g. ``"sha256="``). When provided,
|
||||
the comparison is against the fully-prefixed computed value, so the
|
||||
caller passes the raw header value unchanged. When omitted, an
|
||||
``algo=`` style prefix on the provided signature is auto-stripped.
|
||||
|
||||
Returns:
|
||||
``True`` only if a non-empty signature matches the computed HMAC using a
|
||||
constant-time comparison.
|
||||
"""
|
||||
if not secret or not signature:
|
||||
return False
|
||||
|
||||
secret_bytes = secret.encode("utf-8") if isinstance(secret, str) else secret
|
||||
body_bytes = raw_body if isinstance(raw_body, bytes) else str(raw_body).encode("utf-8")
|
||||
|
||||
try:
|
||||
computed = hmac.new(secret_bytes, body_bytes, algo).hexdigest()
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
provided = signature
|
||||
if prefix:
|
||||
return hmac.compare_digest(f"{prefix}{computed}", provided)
|
||||
|
||||
if "=" in provided and provided.split("=", 1)[0].isalnum():
|
||||
provided = provided.split("=", 1)[1]
|
||||
return hmac.compare_digest(computed, provided)
|
||||
|
||||
|
||||
@dataclass
|
||||
class HookConfig:
|
||||
"""Declarative definition of an inbound event trigger.
|
||||
@@ -172,6 +224,20 @@ class HookConfig:
|
||||
message: Template for the message built from the payload.
|
||||
enabled: Whether the hook is active.
|
||||
metadata: Free-form extra settings.
|
||||
secret: Optional HMAC signing secret. When set, the gateway verifies the
|
||||
provider signature over the *raw* body before any agent runs and
|
||||
rejects (401) a missing/invalid signature — fail-closed.
|
||||
signature_header: Header carrying the signature, e.g.
|
||||
``"X-Hub-Signature-256"``.
|
||||
signature_algo: Digest for the HMAC, e.g. ``"sha256"``.
|
||||
signature_prefix: Optional signature prefix, e.g. ``"sha256="``.
|
||||
events: Optional allow-list of event types. A delivery whose event is
|
||||
not listed is acknowledged (200) without running a turn.
|
||||
event_header: Header carrying the event type, e.g. ``"X-GitHub-Event"``.
|
||||
When omitted the event is read from the payload (dotted path) via
|
||||
``resolve_event``.
|
||||
deliver_only: When ``True`` the rendered ``message`` *is* the delivered
|
||||
content, routed straight through ``deliver_to`` with no LLM turn.
|
||||
"""
|
||||
|
||||
path: str
|
||||
@@ -184,6 +250,16 @@ class HookConfig:
|
||||
message: Optional[str] = None
|
||||
enabled: bool = True
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
# Signature verification (over the RAW body, before the agent runs).
|
||||
secret: Optional[str] = None
|
||||
signature_header: Optional[str] = None
|
||||
signature_algo: str = "sha256"
|
||||
signature_prefix: Optional[str] = None
|
||||
# Event filtering.
|
||||
events: Optional[List[str]] = None
|
||||
event_header: Optional[str] = None
|
||||
# Pass-through: deliver the rendered message with no LLM turn.
|
||||
deliver_only: bool = False
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.path = (self.path or "").strip().strip("/")
|
||||
@@ -194,6 +270,87 @@ class HookConfig:
|
||||
f"HookConfig.action must be one of {HookAction.all()}, "
|
||||
f"got {self.action!r}"
|
||||
)
|
||||
if isinstance(self.events, str):
|
||||
self.events = [self.events]
|
||||
# A configured secret with no explicit header would otherwise read no
|
||||
# signature and reject every request. Default to the widely-used
|
||||
# ``X-Hub-Signature-256`` (GitHub/webhook convention) so ``secret`` on
|
||||
# its own is a working, verifying configuration rather than a 401 trap.
|
||||
if self.secret and not self.signature_header:
|
||||
self.signature_header = "X-Hub-Signature-256"
|
||||
|
||||
def verify_signature(
|
||||
self, raw_body: bytes, headers: Dict[str, str]
|
||||
) -> bool:
|
||||
"""Verify the provider HMAC signature over ``raw_body``.
|
||||
|
||||
Returns ``True`` when no ``secret`` is configured (signature checking is
|
||||
opt-in); otherwise delegates to :func:`verify_webhook_signature`, which
|
||||
is fail-closed on a missing/invalid signature.
|
||||
"""
|
||||
if not self.secret:
|
||||
return True
|
||||
signature = None
|
||||
if self.signature_header:
|
||||
lowered = {k.lower(): v for k, v in headers.items()}
|
||||
signature = lowered.get(self.signature_header.lower())
|
||||
return verify_webhook_signature(
|
||||
self.secret,
|
||||
raw_body,
|
||||
signature,
|
||||
algo=self.signature_algo,
|
||||
prefix=self.signature_prefix,
|
||||
)
|
||||
|
||||
def resolve_event(
|
||||
self, payload: Dict[str, Any], headers: Optional[Dict[str, str]] = None
|
||||
) -> str:
|
||||
"""Resolve the event type from a header or the payload.
|
||||
|
||||
Reads ``event_header`` from ``headers`` when configured, else treats
|
||||
``event_header`` as a dotted payload path (defaulting to ``"event"``).
|
||||
Returns the base event name only (e.g. GitHub's ``"issues"``); a
|
||||
payload ``action`` sub-type (``"issues.opened"``) is matched separately
|
||||
by :meth:`event_allowed` so both ``issues`` and ``issues.opened`` work.
|
||||
"""
|
||||
if self.event_header and headers:
|
||||
lowered = {k.lower(): v for k, v in headers.items()}
|
||||
value = lowered.get(self.event_header.lower())
|
||||
if value:
|
||||
return str(value)
|
||||
path = self.event_header or "event"
|
||||
value = _lookup(path, payload)
|
||||
return "" if value is None else str(value)
|
||||
|
||||
def event_allowed(
|
||||
self, payload: Dict[str, Any], headers: Optional[Dict[str, str]] = None
|
||||
) -> bool:
|
||||
"""Whether the delivery's event passes the configured ``events`` filter.
|
||||
|
||||
Returns ``True`` when no filter is set. The resolved base event (e.g.
|
||||
GitHub's ``"issues"``) matches a listed name that is equal to it or is
|
||||
namespaced under it (``"issues.opened"``); in the namespaced case the
|
||||
payload's ``action`` (when present) must equal the sub-type, so
|
||||
``events: [issues.opened]`` accepts an ``issues`` delivery only when
|
||||
``action == "opened"``.
|
||||
"""
|
||||
if not self.events:
|
||||
return True
|
||||
event = self.resolve_event(payload, headers)
|
||||
if not event:
|
||||
return False
|
||||
action = payload.get("action") if isinstance(payload, dict) else None
|
||||
for allowed in self.events:
|
||||
if event == allowed:
|
||||
return True
|
||||
base, sep, sub = allowed.partition(".")
|
||||
# Namespaced sub-type (e.g. ``issues.opened``): fail-closed — the
|
||||
# payload must actually carry the matching ``action``. A delivery
|
||||
# that omits ``action`` is NOT admitted, so an ``issues`` event
|
||||
# cannot slip through a filter that only allows ``issues.opened``.
|
||||
if sep and base == event and action is not None and str(action) == sub:
|
||||
return True
|
||||
return False
|
||||
|
||||
@property
|
||||
def route(self) -> str:
|
||||
@@ -223,7 +380,7 @@ class HookConfig:
|
||||
return render_template(self.message, payload)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to a dictionary (hides the auth secret)."""
|
||||
"""Convert to a dictionary (hides the auth/signing secrets)."""
|
||||
return {
|
||||
"path": self.path,
|
||||
"agent": self.agent,
|
||||
@@ -235,6 +392,13 @@ class HookConfig:
|
||||
"message": self.message,
|
||||
"enabled": self.enabled,
|
||||
"metadata": dict(self.metadata),
|
||||
"secret": "***" if self.secret else None,
|
||||
"signature_header": self.signature_header,
|
||||
"signature_algo": self.signature_algo,
|
||||
"signature_prefix": self.signature_prefix,
|
||||
"events": list(self.events) if self.events else None,
|
||||
"event_header": self.event_header,
|
||||
"deliver_only": self.deliver_only,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
@@ -251,6 +415,13 @@ class HookConfig:
|
||||
"message",
|
||||
"enabled",
|
||||
"metadata",
|
||||
"secret",
|
||||
"signature_header",
|
||||
"signature_algo",
|
||||
"signature_prefix",
|
||||
"events",
|
||||
"event_header",
|
||||
"deliver_only",
|
||||
}
|
||||
return cls(
|
||||
path=data.get("path", ""),
|
||||
@@ -262,6 +433,13 @@ class HookConfig:
|
||||
deliver_to=data.get("deliver_to"),
|
||||
message=data.get("message"),
|
||||
enabled=data.get("enabled", True),
|
||||
secret=data.get("secret"),
|
||||
signature_header=data.get("signature_header"),
|
||||
signature_algo=data.get("signature_algo", "sha256"),
|
||||
signature_prefix=data.get("signature_prefix"),
|
||||
events=data.get("events"),
|
||||
event_header=data.get("event_header"),
|
||||
deliver_only=data.get("deliver_only", False),
|
||||
metadata={
|
||||
**(data.get("metadata") or {}),
|
||||
**{k: v for k, v in data.items() if k not in known},
|
||||
|
||||
Loaded 100 of 389 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user