mirror of
https://github.com/MervinPraison/PraisonAI.git
synced 2026-11-08 01:08:39 -10:00
Compare commits
227
Commits
No files matched your search
@@ -40,6 +40,62 @@ assert('claude final reply detected', mg.isClaudeFinalReplyComment(withClaudeRep
|
||||
|
||||
assert('cancelled detect-and-trigger does not block', mg.OPTIONAL_CANCELLED_CHECKS.has('detect-and-trigger'));
|
||||
|
||||
const coreGreenRuns = [
|
||||
{ name: 'test-core', status: 'completed', conclusion: 'success' },
|
||||
{ name: 'test-core (cli)', status: 'completed', conclusion: 'success' },
|
||||
{ name: 'smoke', status: 'completed', conclusion: 'cancelled' },
|
||||
{ name: 'test-windows', status: 'completed', conclusion: 'cancelled' },
|
||||
];
|
||||
assert('core green allows cancelled smoke', mg.isAcceptableCheckConclusion(
|
||||
{ name: 'smoke', status: 'completed', conclusion: 'cancelled' },
|
||||
coreGreenRuns
|
||||
));
|
||||
assert('core green allows cancelled test-windows', mg.isAcceptableCheckConclusion(
|
||||
{ name: 'test-windows', status: 'completed', conclusion: 'cancelled' },
|
||||
coreGreenRuns
|
||||
));
|
||||
assert('cancelled smoke blocks when core missing', !mg.isAcceptableCheckConclusion(
|
||||
{ name: 'smoke', status: 'completed', conclusion: 'cancelled' },
|
||||
[{ name: 'smoke', status: 'completed', conclusion: 'cancelled' }]
|
||||
));
|
||||
assert('cancelled smoke blocks when core failed', !mg.isAcceptableCheckConclusion(
|
||||
{ name: 'smoke', status: 'completed', conclusion: 'cancelled' },
|
||||
[
|
||||
{ name: 'test-core', status: 'completed', conclusion: 'failure' },
|
||||
{ name: 'smoke', status: 'completed', conclusion: 'cancelled' },
|
||||
]
|
||||
));
|
||||
assert('bestRunsByName prefers success over cancelled', (() => {
|
||||
const best = mg.bestRunsByName([
|
||||
{ name: 'smoke', conclusion: 'cancelled' },
|
||||
{ name: 'smoke', conclusion: 'success' },
|
||||
]);
|
||||
return best.length === 1 && best[0].conclusion === 'success';
|
||||
})());
|
||||
assert('bestRunsByName prefers pending over completed cancelled', (() => {
|
||||
const best = mg.bestRunsByName([
|
||||
{ name: 'smoke', status: 'completed', conclusion: 'cancelled' },
|
||||
{ name: 'smoke', status: 'in_progress', conclusion: null },
|
||||
]);
|
||||
return best.length === 1 && best[0].status === 'in_progress';
|
||||
})());
|
||||
assert('bestRunsByName keeps pending regardless of order', (() => {
|
||||
const best = mg.bestRunsByName([
|
||||
{ name: 'smoke', status: 'in_progress', conclusion: null },
|
||||
{ name: 'smoke', status: 'completed', conclusion: 'success' },
|
||||
]);
|
||||
return best.length === 1 && best[0].status === 'in_progress';
|
||||
})());
|
||||
assert('allChecksGreenOnSha style: pending re-run blocks despite core green', (() => {
|
||||
const runs = mg.bestRunsByName([
|
||||
{ name: 'test-core', status: 'completed', conclusion: 'success' },
|
||||
{ name: 'smoke', status: 'completed', conclusion: 'cancelled' },
|
||||
{ name: 'smoke', status: 'queued', conclusion: null },
|
||||
]);
|
||||
const smoke = runs.find((r) => r.name === 'smoke');
|
||||
return smoke.status !== 'completed';
|
||||
})());
|
||||
|
||||
// Stale-FINAL recovery guards (PR #2560 push loop)
|
||||
const nowMs = Date.now();
|
||||
const iso = (ms) => new Date(ms).toISOString();
|
||||
|
||||
@@ -44,6 +44,8 @@ const BLOCK_LABELS = new Set([
|
||||
const MERGE_READY_LABEL = 'pipeline/merge-ready';
|
||||
/** Superseded concurrency runs; must not block merge when real tests passed. */
|
||||
const OPTIONAL_CANCELLED_CHECKS = new Set(['detect-and-trigger']);
|
||||
/** Cancelled smoke/windows after timeout are non-blocking when core shards passed on HEAD. */
|
||||
const OPTIONAL_CANCELLED_WHEN_CORE_GREEN = new Set(['smoke', 'test-windows']);
|
||||
const BOT_REVIEWER_PATTERNS = [
|
||||
'coderabbit',
|
||||
'qodo',
|
||||
@@ -325,8 +327,70 @@ function listFailedChecksOnSha(runs) {
|
||||
});
|
||||
}
|
||||
|
||||
function isCoreTestRun(run) {
|
||||
const name = run?.name || '';
|
||||
return name === 'test-core' || name.startsWith('test-core ') || name === 'test-core-collect';
|
||||
}
|
||||
|
||||
function coreTestsGreenOnRuns(runs) {
|
||||
const coreRuns = (runs || []).filter(isCoreTestRun);
|
||||
if (coreRuns.length === 0) return false;
|
||||
return coreRuns.every(
|
||||
(run) =>
|
||||
run.status === 'completed' &&
|
||||
['success', 'neutral', 'skipped'].includes(run.conclusion)
|
||||
);
|
||||
}
|
||||
|
||||
function checkConclusionRank(conclusion) {
|
||||
if (conclusion === 'success') return 4;
|
||||
if (conclusion === 'neutral') return 3;
|
||||
if (conclusion === 'skipped') return 2;
|
||||
if (conclusion === 'cancelled') return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
function isPendingRun(run) {
|
||||
return !!(run && run.status && run.status !== 'completed');
|
||||
}
|
||||
|
||||
function bestRunsByName(runs) {
|
||||
const byName = new Map();
|
||||
for (const run of runs || []) {
|
||||
if (!run) continue;
|
||||
const existing = byName.get(run.name);
|
||||
if (!existing) {
|
||||
byName.set(run.name, run);
|
||||
continue;
|
||||
}
|
||||
const runPending = isPendingRun(run);
|
||||
const existingPending = isPendingRun(existing);
|
||||
if (existingPending) continue;
|
||||
if (
|
||||
runPending ||
|
||||
checkConclusionRank(run.conclusion) > checkConclusionRank(existing.conclusion)
|
||||
) {
|
||||
byName.set(run.name, run);
|
||||
}
|
||||
}
|
||||
return [...byName.values()];
|
||||
}
|
||||
|
||||
function isAcceptableCheckConclusion(run, runs) {
|
||||
if (['success', 'neutral', 'skipped'].includes(run.conclusion)) return true;
|
||||
if (run.conclusion === 'cancelled' && OPTIONAL_CANCELLED_CHECKS.has(run.name)) return true;
|
||||
if (
|
||||
run.conclusion === 'cancelled' &&
|
||||
OPTIONAL_CANCELLED_WHEN_CORE_GREEN.has(run.name) &&
|
||||
coreTestsGreenOnRuns(runs)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async function allChecksGreenOnSha(github, owner, repo, sha, core) {
|
||||
const runs = await listChecksOnSha(github, owner, repo, sha);
|
||||
const runs = bestRunsByName(await listChecksOnSha(github, owner, repo, sha));
|
||||
if (runs.length === 0) {
|
||||
core?.info?.(`No check runs on ${sha.slice(0, 7)} — allowing (e.g. docs-only PR)`);
|
||||
return true;
|
||||
@@ -336,13 +400,17 @@ async function allChecksGreenOnSha(github, owner, repo, sha, core) {
|
||||
core?.info?.(`Check pending: ${run.name} (${run.status})`);
|
||||
return false;
|
||||
}
|
||||
const ok =
|
||||
['success', 'neutral', 'skipped'].includes(run.conclusion) ||
|
||||
(run.conclusion === 'cancelled' && OPTIONAL_CANCELLED_CHECKS.has(run.name));
|
||||
if (!ok) {
|
||||
if (!isAcceptableCheckConclusion(run, runs)) {
|
||||
core?.info?.(`Check failed: ${run.name} (${run.conclusion})`);
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
run.conclusion === 'cancelled' &&
|
||||
OPTIONAL_CANCELLED_WHEN_CORE_GREEN.has(run.name) &&
|
||||
coreTestsGreenOnRuns(runs)
|
||||
) {
|
||||
core?.info?.(`Ignoring cancelled ${run.name} — test-core green on HEAD`);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -855,6 +923,12 @@ module.exports = {
|
||||
finalClaudeCompletedOnSha,
|
||||
getMergeState,
|
||||
OPTIONAL_CANCELLED_CHECKS,
|
||||
OPTIONAL_CANCELLED_WHEN_CORE_GREEN,
|
||||
isCoreTestRun,
|
||||
coreTestsGreenOnRuns,
|
||||
isPendingRun,
|
||||
bestRunsByName,
|
||||
isAcceptableCheckConclusion,
|
||||
listChecksOnSha,
|
||||
listFailedChecksOnSha,
|
||||
allChecksGreenOnSha,
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
name: Terminal-Bench Smoke
|
||||
|
||||
# Manual (and optional weekly) smoke run of the `praisonai code` assistant on a
|
||||
# small verified Terminal-Bench 2.1 subset via the Harbor harness.
|
||||
# Never runs on PRs — it costs money (LLM calls) and is inherently flaky.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 6 * * 1" # Mondays 06:00 UTC
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
smoke:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install Harbor + PraisonAI
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install harbor
|
||||
pip install -e src/praisonai-agents -e src/praisonai
|
||||
|
||||
- name: Run Terminal-Bench 2.1 smoke subset
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
PYTHONPATH: .
|
||||
run: |
|
||||
harbor run -c examples/terminal_bench/job_code_smoke.yaml || true
|
||||
|
||||
- name: Upload Harbor results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: harbor-results
|
||||
path: |
|
||||
runs/
|
||||
**/results.json
|
||||
if-no-files-found: warn
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
# ============================================
|
||||
smoke:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 8
|
||||
timeout-minutes: 12
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
|
||||
@@ -20,7 +20,7 @@ on:
|
||||
jobs:
|
||||
test-windows:
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 5
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
OPENAI_MODEL_NAME: gpt-4o-mini
|
||||
|
||||
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=4.6.152" \
|
||||
"praisonai>=4.6.155" \
|
||||
"praisonai[chat]" \
|
||||
"embedchain[github,youtube]"
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=4.6.152" \
|
||||
"praisonai>=4.6.155" \
|
||||
"praisonai[ui]" \
|
||||
"praisonai[chat]" \
|
||||
"praisonai[realtime]" \
|
||||
|
||||
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=4.6.152" \
|
||||
"praisonai>=4.6.155" \
|
||||
"praisonai[ui]" \
|
||||
"praisonai[crewai]"
|
||||
|
||||
|
||||
Whitespace-only changes.
@@ -19,7 +19,9 @@ def run_doctor_ci():
|
||||
[sys.executable, "-m", "praisonai", "doctor", "ci"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=120,
|
||||
)
|
||||
|
||||
# Parse JSON output
|
||||
|
||||
@@ -149,21 +149,32 @@ def main():
|
||||
|
||||
print(f"\nOriginal document length: {len(content)} characters")
|
||||
|
||||
chunks = [{"text": content, "metadata": {"source": doc_path}}]
|
||||
|
||||
# Estimate original tokens (~4 chars/token) so each ratio maps to a
|
||||
# distinct token budget — compress() truncates to target_tokens.
|
||||
original_tokens = max(1, len(content) // 4)
|
||||
|
||||
# Compress with different ratios
|
||||
for ratio in [0.3, 0.5, 0.7]:
|
||||
compressor = ContextCompressor(
|
||||
max_tokens=2000,
|
||||
target_ratio=ratio,
|
||||
target_tokens = max(1, int(original_tokens * ratio))
|
||||
compressor = ContextCompressor(verbose=False)
|
||||
result = compressor.compress(
|
||||
chunks,
|
||||
query="security verification code",
|
||||
target_tokens=target_tokens,
|
||||
compression_ratio=ratio,
|
||||
)
|
||||
result = compressor.compress([content], query="security verification code")
|
||||
|
||||
print(f"\nCompression ratio {ratio}:")
|
||||
print(f" Original tokens: {result.original_tokens}")
|
||||
print(f" Compressed tokens: {result.compressed_tokens}")
|
||||
print(f" Actual ratio: {result.compressed_tokens / max(result.original_tokens, 1):.2f}")
|
||||
print(f" Actual ratio: {result.compression_ratio:.2f}")
|
||||
|
||||
# Check if secret code is preserved
|
||||
compressed_text = " ".join(result.chunks) if result.chunks else ""
|
||||
compressed_text = " ".join(
|
||||
c.get("text", "") for c in result.chunks
|
||||
) if result.chunks else ""
|
||||
if SECRET_CODE in compressed_text:
|
||||
print(f" ✅ Secret code preserved in compressed output")
|
||||
else:
|
||||
|
||||
@@ -65,7 +65,6 @@ def main():
|
||||
name="Policy_Expert",
|
||||
instructions="You are a policy expert.",
|
||||
knowledge=[temp_dir],
|
||||
agent_id="policy_agent_v1", # Knowledge scoped to this agent
|
||||
)
|
||||
|
||||
response = shared_agent.chat("What are the expense report rules?")
|
||||
@@ -81,7 +80,6 @@ def main():
|
||||
instructions="You are a personal HR assistant.",
|
||||
knowledge=[temp_dir],
|
||||
memory={"user_id": "user_bob"},
|
||||
agent_id="hr_bot_v2",
|
||||
)
|
||||
|
||||
response = combined_agent.chat("What training is required?")
|
||||
|
||||
@@ -13,7 +13,8 @@ Expected Output:
|
||||
Agent responds with state persisted to Redis
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent, db
|
||||
from praisonaiagents import Agent
|
||||
from praisonaiagents.db import db
|
||||
|
||||
print("=== Redis State Store (Agent-First) ===")
|
||||
|
||||
|
||||
@@ -61,7 +61,7 @@ def demonstrate_collision_problem():
|
||||
_reset_default_registry()
|
||||
|
||||
# Agent 1 registers "custom" provider
|
||||
register_llm_provider("custom", Agent1Provider)
|
||||
register_llm_provider("custom", Agent1Provider, override=True)
|
||||
print("Agent 1 registered 'custom' provider")
|
||||
|
||||
# Agent 2 tries to register same name - ERROR!
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
# praisonai: skip=true
|
||||
# Uses removed praisonaiagents.tools exports; pending example refresh
|
||||
from praisonaiagents import Agent, Tools
|
||||
from praisonaiagents.tools import read_csv, read_excel, write_csv, write_excel, filter_data, get_summary, group_by, pivot_table
|
||||
import os
|
||||
@@ -7,4 +9,4 @@ agent.start(f"""
|
||||
Read the data from the csv file {os.path.join(os.path.dirname(__file__), "tesla-stock-price.csv")}
|
||||
Analyse the data and give me the insights
|
||||
read_csv to read the file
|
||||
""")
|
||||
""")
|
||||
@@ -1,5 +1,7 @@
|
||||
# praisonai: skip=true
|
||||
# Uses removed praisonaiagents.tools exports; pending example refresh
|
||||
from praisonaiagents import Agent, Tools
|
||||
from praisonaiagents.tools import get_stock_price, get_stock_info, get_historical_data
|
||||
|
||||
agent = Agent(instructions="You are a Research Agent", tools=[get_stock_price, get_stock_info, get_historical_data])
|
||||
agent.start("Understand current stock price and historical data of Apple and Google. Tell me if I can invest in them")
|
||||
agent.start("Understand current stock price and historical data of Apple and Google. Tell me if I can invest in them")
|
||||
@@ -135,7 +135,7 @@ def example_auto_agents_multi_provider():
|
||||
from praisonaiagents.agents import AutoAgents
|
||||
|
||||
# Create AutoAgents that will automatically assign appropriate models
|
||||
auto_agents = AutoAgentTeam(
|
||||
auto_agents = AutoAgents(
|
||||
instructions="Create a market research report on electric vehicles. Include data analysis, competitor analysis, and future projections.",
|
||||
max_agents=3,
|
||||
llm="gpt-4o-mini", # Default model for agent generation
|
||||
@@ -144,6 +144,7 @@ def example_auto_agents_multi_provider():
|
||||
|
||||
# After agents are created, upgrade them to multi-model agents
|
||||
multi_model_agents = []
|
||||
wrapper_by_original = {}
|
||||
for agent in auto_agents.agents:
|
||||
# Convert regular agents to multi-model agents
|
||||
multi_agent = RouterAgent(
|
||||
@@ -155,9 +156,18 @@ def example_auto_agents_multi_provider():
|
||||
models=["gpt-4o-mini", "gemini/gemini-1.5-flash", "claude-3-haiku-20240307", "gpt-4o-mini"],
|
||||
routing_strategy="auto")
|
||||
multi_model_agents.append(multi_agent)
|
||||
wrapper_by_original[id(agent)] = multi_agent
|
||||
|
||||
# Update the agents in the AutoAgents instance
|
||||
auto_agents.agents = multi_model_agents
|
||||
|
||||
# Rebind each task's agent to its wrapper — AutoAgents stores a direct agent
|
||||
# reference on every Task and execution reads task.agent, so replacing only
|
||||
# auto_agents.agents would leave the workflow bound to the original agents.
|
||||
for task in getattr(auto_agents, "tasks", []) or []:
|
||||
wrapper = wrapper_by_original.get(id(task.agent))
|
||||
if wrapper is not None:
|
||||
task.agent = wrapper
|
||||
|
||||
# Run the auto-generated workflow
|
||||
results = auto_agents.start()
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
# praisonai: skip=true
|
||||
# Uses removed praisonaiagents.tools exports; pending example refresh
|
||||
from praisonaiagents import Agent, Task, AgentTeam
|
||||
from praisonaiagents.tools import wiki_search, wiki_summary, wiki_page, wiki_random, wiki_language
|
||||
|
||||
@@ -13,4 +15,4 @@ agent.start(
|
||||
"First search the history of AI"
|
||||
"Read the page of the history of AI"
|
||||
"Get the summary of the page"
|
||||
)
|
||||
)
|
||||
@@ -1,3 +1,5 @@
|
||||
# praisonai: skip=true
|
||||
# Requires local audio.mp3 fixture
|
||||
from praisonaiagents import AudioAgent
|
||||
|
||||
agent = AudioAgent(llm="groq/whisper-large-v3") # 10x faster
|
||||
@@ -5,3 +7,4 @@ text = agent.transcribe("audio.mp3", language="en")
|
||||
print(text)
|
||||
|
||||
# Models: openai/whisper-1, groq/whisper-large-v3, deepgram/nova-2
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
# praisonai: skip=true
|
||||
# Requires local audio.mp3 fixture
|
||||
# Speech-to-Text with Groq (fastest)
|
||||
# Requires: export GROQ_API_KEY=your-key
|
||||
|
||||
@@ -6,3 +8,4 @@ from praisonaiagents import AudioAgent
|
||||
agent = AudioAgent(llm="groq/whisper-large-v3")
|
||||
text = agent.listen("audio.mp3") # Replace with your audio file
|
||||
print(text)
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# praisonai: skip=true
|
||||
# Requires local audio.mp3 fixture
|
||||
from praisonaiagents import AudioAgent
|
||||
|
||||
agent = AudioAgent(llm="deepgram/nova-2")
|
||||
text = agent.listen("audio.mp3")
|
||||
print(text)
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# praisonai: skip=true
|
||||
# Requires local audio.mp3 fixture
|
||||
from praisonaiagents import AudioAgent
|
||||
|
||||
agent = AudioAgent(llm="groq/whisper-large-v3")
|
||||
text = agent.listen("audio.mp3")
|
||||
print(text)
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# praisonai: skip=true
|
||||
# Requires local audio.mp3 fixture
|
||||
from praisonaiagents import AudioAgent
|
||||
|
||||
agent = AudioAgent(llm="openai/whisper-1")
|
||||
text = agent.listen("audio.mp3")
|
||||
print(text)
|
||||
|
||||
@@ -18,7 +18,7 @@ from praisonaiagents import AutoAgents
|
||||
from praisonaiagents.tools import duckduckgo
|
||||
|
||||
# Basic usage - AutoAgents analyzes complexity and creates optimal agents
|
||||
agents = AutoAgentTeam(
|
||||
agents = AutoAgents(
|
||||
instructions="Search for information about AI Agents",
|
||||
tools=[duckduckgo],
|
||||
process="sequential", # or "hierarchical"
|
||||
|
||||
@@ -102,7 +102,14 @@ def main():
|
||||
# Print results
|
||||
print("\nAutonomous Agent Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
task_name = result.description
|
||||
print(f"\nTask: {task_name}")
|
||||
|
||||
@@ -64,6 +64,13 @@ results = workflow.start()
|
||||
|
||||
# Print results
|
||||
print("\nEvaluator-Optimizer Results:")
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"Task {task_id}: {result.raw}")
|
||||
@@ -1,4 +1,4 @@
|
||||
from praisonaiagents.agents.agents import Agent, Task, Agents
|
||||
from praisonaiagents.agents.agents import Agent, Task, Agents, AgentTeam
|
||||
from praisonaiagents.tools import duckduckgo
|
||||
|
||||
# Test facts
|
||||
|
||||
@@ -104,6 +104,13 @@ results = workflow.start()
|
||||
|
||||
# Print results
|
||||
print("\nOrchestrator-Workers Results:")
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"Task {task_id}: {result.raw}")
|
||||
@@ -86,7 +86,14 @@ async def main():
|
||||
|
||||
# Print results
|
||||
print("\nParallel Processing Results:")
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"Task {task_id}: {result.raw}")
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from praisonaiagents.agent import Agent
|
||||
from praisonaiagents.task import Task
|
||||
from praisonaiagents.agents import Agents
|
||||
from praisonaiagents.agents import Agents, AgentTeam
|
||||
from typing import List, Dict
|
||||
import time
|
||||
|
||||
@@ -76,6 +76,13 @@ results = workflow.start(return_dict=True)
|
||||
|
||||
# Print results
|
||||
print("\nWorkflow Results:")
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"Task {task_id}: {result.raw}")
|
||||
@@ -121,7 +121,14 @@ def main():
|
||||
# Print results
|
||||
print("\nAdaptive Learning Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -204,7 +204,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nClimate Impact Analysis Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -97,7 +97,14 @@ def main():
|
||||
# Print results
|
||||
print("\nCode Review Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -222,7 +222,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nCryptography Validation Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -128,7 +128,14 @@ def main():
|
||||
# Print results
|
||||
print("\nCustomer Service Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -208,7 +208,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nMarket Making Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -401,7 +401,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nDisaster Recovery Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -128,7 +128,14 @@ def main():
|
||||
# Print results
|
||||
print("\nEmergency Response Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -125,7 +125,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nFraud Detection Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -153,7 +153,14 @@ def main():
|
||||
# Print results
|
||||
print("\nDiagnosis Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -206,7 +206,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nProtocol Generation Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -150,7 +150,14 @@ def main():
|
||||
# Print results
|
||||
print("\nContent Generation Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -191,7 +191,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nArchitecture Search Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -169,7 +169,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nMaintenance Planning Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -186,7 +186,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nOptimization Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -181,7 +181,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nResearch Analysis Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -175,7 +175,14 @@ def main():
|
||||
# Print results
|
||||
print("\nOptimization Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -220,7 +220,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nResource Optimization Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -102,7 +102,14 @@ def main():
|
||||
# Print results
|
||||
print("\nRisk Management Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -184,7 +184,14 @@ async def main():
|
||||
# Print results
|
||||
print("\nVulnerability Detection Results:")
|
||||
print("=" * 50)
|
||||
for task_id, result in results["task_results"].items():
|
||||
task_results = (
|
||||
results.get("task_results", {})
|
||||
if isinstance(results, dict)
|
||||
else {}
|
||||
)
|
||||
if not task_results and isinstance(results, str):
|
||||
print(f"\nWorkflow output:\n{results}")
|
||||
for task_id, result in task_results.items():
|
||||
if result:
|
||||
print(f"\nTask: {task_id}")
|
||||
print(f"Result: {result.raw}")
|
||||
|
||||
@@ -4,7 +4,7 @@ Agentic Loop with CSV Workflow Example
|
||||
Demonstrates iterating over a CSV file with an agent processing each row.
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent, Workflow
|
||||
from praisonaiagents import Agent, AgentFlow, Workflow
|
||||
from praisonaiagents.workflows import loop
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
@@ -5,7 +5,7 @@ Demonstrates the evaluator-optimizer pattern where an agent
|
||||
generates content and another evaluates it, repeating until approved.
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent, Workflow
|
||||
from praisonaiagents import Agent, AgentFlow, Workflow
|
||||
from praisonaiagents.workflows import repeat
|
||||
|
||||
# Create generator agent
|
||||
|
||||
@@ -12,7 +12,7 @@ Prerequisites:
|
||||
--port 6060
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent, Workflow
|
||||
from praisonaiagents import Agent, AgentFlow, Workflow
|
||||
|
||||
ROUTELLM_URL = "http://localhost:6060/v1"
|
||||
|
||||
|
||||
@@ -78,7 +78,7 @@ def main():
|
||||
}''')
|
||||
print("\nPress Ctrl+C to stop")
|
||||
|
||||
server.run(transport="sse", host="0.0.0.0", port=8081)
|
||||
server.run_sse(port=8081)
|
||||
|
||||
|
||||
def run_manual_server():
|
||||
|
||||
@@ -1,18 +1,40 @@
|
||||
# Terminal-Bench 2.0 Integration with PraisonAI
|
||||
# Terminal-Bench 2.1 Integration with PraisonAI
|
||||
|
||||
This directory contains examples for integrating PraisonAI Agents with **Terminal-Bench 2.0** via the **Harbor framework** for AI agent benchmarking.
|
||||
This directory contains adapters for benchmarking PraisonAI on **Terminal-Bench 2.1** via the **Harbor framework**. Everything lives in the examples/benchmark layer + CI — there are **zero changes to the `praisonaiagents` / `praisonai` SDK surface**.
|
||||
|
||||
## Overview
|
||||
|
||||
[Terminal-Bench 2.0](https://tbench.ai) is a Stanford/Laude Institute benchmark that has become the gold standard for evaluating AI coding agents in real terminal environments. The [Harbor framework](https://harborframework.com) is the official evaluation harness that abstracts container lifecycle, parallelization, and cloud providers.
|
||||
[Terminal-Bench 2.1](https://www.tbench.ai/docs/run-terminal-bench-2-1) is the current release of the Laude Institute benchmark for evaluating AI coding agents in real terminal environments. The [Harbor framework](https://github.com/laude-institute/harbor) is the official evaluation harness that abstracts container lifecycle, parallelization, and cloud providers.
|
||||
|
||||
## Setup
|
||||
|
||||
Install Harbor and PraisonAI into **one venv** and run from the repo root with `PYTHONPATH=.` so the adapter import path resolves:
|
||||
|
||||
```bash
|
||||
pip install harbor
|
||||
pip install -e src/praisonai-agents -e src/praisonai
|
||||
export PYTHONPATH=.
|
||||
```
|
||||
|
||||
## Integration Types
|
||||
|
||||
### 0. Code Agent (headline — benchmarks `praisonai code`)
|
||||
- **File**: `praisonai_code_agent.py`
|
||||
- **Purpose**: Installs and drives the terminal-native `praisonai code` assistant headlessly inside the container.
|
||||
- **Run**:
|
||||
```bash
|
||||
PYTHONPATH=. harbor run -d terminal-bench/terminal-bench-2-1 \
|
||||
--agent "examples.terminal_bench.praisonai_code_agent:PraisonAICodeAgent" \
|
||||
-m openai/gpt-4o-mini \
|
||||
--ae OPENAI_API_KEY=$OPENAI_API_KEY \
|
||||
-n 4
|
||||
```
|
||||
|
||||
### 1. External Agent (Direct Agent Class)
|
||||
- **File**: `praisonai_external_agent.py`
|
||||
- **Purpose**: External agent that uses direct `Agent()` class instantiation
|
||||
- **Usage**: Run with `--agent-import-path` flag
|
||||
- **Approach**: Uses `praisonaiagents.Agent` directly with `execute_command` tool
|
||||
- **Usage**: Run with `--agent "module:Class"`
|
||||
- **Approach**: Uses `praisonaiagents.Agent` directly with a bash tool bridged to Harbor's `exec()`
|
||||
|
||||
### 2. Wrapper Agent (CLI-Based)
|
||||
- **File**: `praisonai_wrapper_agent.py`
|
||||
@@ -42,12 +64,12 @@ pip install praisonaiagents[tools]
|
||||
|
||||
```bash
|
||||
# Test with oracle agent first
|
||||
harbor run -d terminal-bench/terminal-bench-2 -a oracle
|
||||
harbor run -d terminal-bench/terminal-bench-2-1 -a oracle
|
||||
|
||||
# Run PraisonAI external agent (uses direct Agent() class)
|
||||
harbor run -d terminal-bench/terminal-bench-2 \
|
||||
--agent-import-path examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent \
|
||||
--model openai/gpt-4o \
|
||||
PYTHONPATH=. harbor run -d terminal-bench/terminal-bench-2-1 \
|
||||
--agent "examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent" \
|
||||
--model openai/gpt-4o-mini \
|
||||
--ae OPENAI_API_KEY=$OPENAI_API_KEY \
|
||||
-n 4
|
||||
```
|
||||
@@ -56,9 +78,9 @@ harbor run -d terminal-bench/terminal-bench-2 \
|
||||
|
||||
```bash
|
||||
# Run PraisonAI wrapper agent (uses `praisonai "TASK"` CLI pattern)
|
||||
harbor run -d terminal-bench/terminal-bench-2 \
|
||||
--agent-import-path examples.terminal_bench.praisonai_wrapper_agent:PraisonAIWrapperAgent \
|
||||
--model openai/gpt-4o \
|
||||
PYTHONPATH=. harbor run -d terminal-bench/terminal-bench-2-1 \
|
||||
--agent "examples.terminal_bench.praisonai_wrapper_agent:PraisonAIWrapperAgent" \
|
||||
--model openai/gpt-4o-mini \
|
||||
--ae OPENAI_API_KEY=$OPENAI_API_KEY \
|
||||
-n 4
|
||||
```
|
||||
@@ -66,9 +88,9 @@ harbor run -d terminal-bench/terminal-bench-2 \
|
||||
### Running on Cloud (Daytona/E2B/Modal)
|
||||
|
||||
```bash
|
||||
harbor run -d terminal-bench/terminal-bench-2 \
|
||||
--agent-import-path examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent \
|
||||
--model openai/gpt-4o \
|
||||
PYTHONPATH=. harbor run -d terminal-bench/terminal-bench-2-1 \
|
||||
--agent "examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent" \
|
||||
--model openai/gpt-4o-mini \
|
||||
--env daytona -n 32 \
|
||||
--ae OPENAI_API_KEY=$OPENAI_API_KEY
|
||||
```
|
||||
@@ -100,6 +122,9 @@ harbor run -d terminal-bench/terminal-bench-2 \
|
||||
## Files
|
||||
|
||||
- `README.md` - This documentation
|
||||
- `praisonai_code_agent.py` - **Headline** adapter benchmarking `praisonai code`
|
||||
- `job_code_smoke.yaml` - Smoke-test job config for the code agent
|
||||
- `RESULTS.md` - Recorded pass-rates
|
||||
- `praisonai_external_agent.py` - External agent (direct Agent class)
|
||||
- `praisonai_wrapper_agent.py` - Wrapper agent (CLI-based approach)
|
||||
- `praisonai_installed_agent.py` - Installed agent implementation
|
||||
@@ -127,7 +152,7 @@ Each task provides:
|
||||
|
||||
## Contributing
|
||||
|
||||
1. Test changes with oracle agent first: `harbor run -d terminal-bench/terminal-bench-2 -a oracle`
|
||||
1. Test changes with oracle agent first: `harbor run -d terminal-bench/terminal-bench-2-1 -a oracle`
|
||||
2. Run real agentic tests to ensure end-to-end functionality
|
||||
3. Follow PraisonAI's AGENTS.md architecture guidelines
|
||||
4. Add both unit tests and integration tests
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
# Terminal-Bench 2.1 Results — PraisonAI
|
||||
|
||||
Pass-rates from the Harbor smoke workflow
|
||||
(`.github/workflows/terminal-bench-smoke.yml`, manual dispatch). The full 2.1
|
||||
dataset (89 tasks) can be scored by dropping the `task_names` filter in
|
||||
`job.yaml`.
|
||||
|
||||
| Date | Agent | Model | Tasks | Pass rate | Harbor artifact |
|
||||
|------|-------|-------|-------|-----------|-----------------|
|
||||
| _pending first run_ | `praisonai code` | `openai/gpt-4o-mini` | smoke (3) | — | — |
|
||||
|
||||
## How scores are produced
|
||||
|
||||
```bash
|
||||
PYTHONPATH=. harbor run -c examples/terminal_bench/job_code_smoke.yaml
|
||||
```
|
||||
|
||||
The workflow uploads the Harbor results directory as a CI artifact and appends
|
||||
the pass-rate row above.
|
||||
Whitespace-only changes.
@@ -1,83 +1,20 @@
|
||||
# Harbor Job Configuration for PraisonAI on Terminal-Bench 2.0
|
||||
#
|
||||
# Usage:
|
||||
# harbor run -c examples/terminal_bench/job.yaml
|
||||
# Harbor Job Configuration — PraisonAI external (SDK-level) agent on Terminal-Bench 2.1
|
||||
#
|
||||
# This configuration runs PraisonAI external agent on Terminal-Bench 2.0 tasks
|
||||
# with 8 concurrent trials using GPT-4o.
|
||||
|
||||
# Dataset: Terminal-Bench 2.0 (89 carefully curated terminal tasks)
|
||||
dataset: terminal-bench/terminal-bench-2
|
||||
|
||||
# Agent configuration
|
||||
agent:
|
||||
# Use import path for external agent
|
||||
import_path: examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent
|
||||
|
||||
# Model configuration
|
||||
model_name: openai/gpt-4o
|
||||
|
||||
# Environment variables (API keys)
|
||||
env:
|
||||
OPENAI_API_KEY: "${OPENAI_API_KEY}"
|
||||
# Optional: Add other API keys if using different models
|
||||
# ANTHROPIC_API_KEY: "${ANTHROPIC_API_KEY}"
|
||||
# GEMINI_API_KEY: "${GEMINI_API_KEY}"
|
||||
|
||||
# Execution configuration
|
||||
n_concurrent: 8 # Run 8 trials in parallel
|
||||
n_attempts: 1 # Single attempt per task (benchmark standard)
|
||||
|
||||
# Optional: Environment configuration for cloud execution
|
||||
# environment:
|
||||
# provider: daytona # Options: local, daytona, e2b, modal, runloop, gke
|
||||
# n_concurrent: 32 # Higher concurrency on cloud
|
||||
|
||||
# Optional: Filter to specific tasks for testing
|
||||
# task_filter:
|
||||
# task_names: ["compile_simple_c", "install_python_package", "create_directory"]
|
||||
|
||||
# Optional: Advanced configuration
|
||||
# timeout_sec: 600 # 10 minute timeout per task
|
||||
# save_logs: true # Save execution logs
|
||||
# save_artifacts: true # Save container artifacts
|
||||
|
||||
---
|
||||
|
||||
# Alternative configuration using wrapper agent (CLI-based approach)
|
||||
# Uses `praisonai "TASK"` pattern instead of direct Agent class
|
||||
# Uncomment to use the wrapper approach:
|
||||
|
||||
# dataset: terminal-bench/terminal-bench-2
|
||||
# Run from the repo root with PYTHONPATH set:
|
||||
# PYTHONPATH=. harbor run -c examples/terminal_bench/job.yaml
|
||||
#
|
||||
# agent:
|
||||
# import_path: examples.terminal_bench.praisonai_wrapper_agent:PraisonAIWrapperAgent
|
||||
# model_name: openai/gpt-4o
|
||||
#
|
||||
# n_concurrent: 8
|
||||
# n_attempts: 1
|
||||
#
|
||||
# env:
|
||||
# OPENAI_API_KEY: "${OPENAI_API_KEY}"
|
||||
# Uses the REAL Harbor JobConfig schema. A single YAML document only —
|
||||
# multiple `---` documents make yaml.safe_load raise ComposerError.
|
||||
|
||||
---
|
||||
datasets:
|
||||
- name: terminal-bench
|
||||
version: "2.1"
|
||||
|
||||
# Alternative configuration using installed agent (requires Harbor integration)
|
||||
# Uncomment this section once PraisonAI is integrated into Harbor's codebase
|
||||
agents:
|
||||
- import_path: examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent
|
||||
model_name: openai/gpt-4o-mini
|
||||
env:
|
||||
OPENAI_API_KEY: "${OPENAI_API_KEY}"
|
||||
|
||||
# dataset: terminal-bench/terminal-bench-2
|
||||
#
|
||||
# agent:
|
||||
# name: praisonai
|
||||
# model_name: openai/gpt-4o
|
||||
# flags:
|
||||
# max_turns: 30
|
||||
# verbose: false
|
||||
# memory: false
|
||||
# auto_approval: true
|
||||
#
|
||||
# n_concurrent: 8
|
||||
# n_attempts: 1
|
||||
#
|
||||
# env:
|
||||
# OPENAI_API_KEY: "${OPENAI_API_KEY}"
|
||||
n_concurrent_trials: 4
|
||||
n_attempts: 1
|
||||
@@ -0,0 +1,28 @@
|
||||
# Harbor Job Configuration — `praisonai code` smoke test on Terminal-Bench 2.1
|
||||
#
|
||||
# Run from the repo root with PYTHONPATH set so the adapter import path resolves:
|
||||
# PYTHONPATH=. harbor run -c examples/terminal_bench/job_code_smoke.yaml
|
||||
#
|
||||
# Schema note: this uses the REAL Harbor JobConfig schema
|
||||
# (datasets/agents/n_concurrent_trials/n_attempts). Unknown keys are silently
|
||||
# ignored by pydantic, so the older dataset:/agent:/n_concurrent: form runs the
|
||||
# default oracle agent on zero tasks — do not use it.
|
||||
|
||||
datasets:
|
||||
- name: terminal-bench
|
||||
version: "2.1"
|
||||
# A small verified subset for a cheap smoke run. Verify against the registry
|
||||
# with `harbor datasets list` before editing.
|
||||
task_names:
|
||||
- hello-world
|
||||
- fix-permissions
|
||||
- csv-to-parquet
|
||||
|
||||
agents:
|
||||
- import_path: examples.terminal_bench.praisonai_code_agent:PraisonAICodeAgent
|
||||
model_name: openai/gpt-4o-mini
|
||||
env:
|
||||
OPENAI_API_KEY: "${OPENAI_API_KEY}"
|
||||
|
||||
n_concurrent_trials: 2
|
||||
n_attempts: 1
|
||||
@@ -1,22 +1,24 @@
|
||||
# Harbor Job Configuration for PraisonAI - 5 Task Test
|
||||
# Run with: harbor run -c examples/terminal_bench/job_test_5.yaml -n 1
|
||||
# Harbor Job Configuration — PraisonAI external agent, small verified subset
|
||||
#
|
||||
# Run from the repo root:
|
||||
# PYTHONPATH=. harbor run -c examples/terminal_bench/job_test_5.yaml
|
||||
#
|
||||
# Real Harbor JobConfig schema. Verify task_names against `harbor datasets list`
|
||||
# before editing.
|
||||
|
||||
dataset: terminal-bench/terminal-bench-2
|
||||
datasets:
|
||||
- name: terminal-bench
|
||||
version: "2.1"
|
||||
task_names:
|
||||
- hello-world
|
||||
- fix-permissions
|
||||
- csv-to-parquet
|
||||
|
||||
agent:
|
||||
import_path: examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent
|
||||
model_name: openai/gpt-4o-mini
|
||||
env:
|
||||
OPENAI_API_KEY: "${OPENAI_API_KEY}"
|
||||
agents:
|
||||
- import_path: examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent
|
||||
model_name: openai/gpt-4o-mini
|
||||
env:
|
||||
OPENAI_API_KEY: "${OPENAI_API_KEY}"
|
||||
|
||||
n_concurrent: 1
|
||||
n_concurrent_trials: 1
|
||||
n_attempts: 1
|
||||
|
||||
# Filter to 5 specific simple tasks
|
||||
task_filter:
|
||||
task_names:
|
||||
- "hello-world"
|
||||
- "fix-permissions"
|
||||
- "create-bucket"
|
||||
- "csv-to-parquet"
|
||||
- "simple-web-scraper"
|
||||
@@ -1,22 +1,24 @@
|
||||
# Harbor Job Configuration for PraisonAI Wrapper - 5 Task Test
|
||||
# Run with: harbor run -c examples/terminal_bench/job_test_5_wrapper.yaml -n 1
|
||||
# Harbor Job Configuration — PraisonAI wrapper (CLI) agent, small verified subset
|
||||
#
|
||||
# Run from the repo root:
|
||||
# PYTHONPATH=. harbor run -c examples/terminal_bench/job_test_5_wrapper.yaml
|
||||
#
|
||||
# Real Harbor JobConfig schema. Verify task_names against `harbor datasets list`
|
||||
# before editing.
|
||||
|
||||
dataset: terminal-bench/terminal-bench-2
|
||||
datasets:
|
||||
- name: terminal-bench
|
||||
version: "2.1"
|
||||
task_names:
|
||||
- hello-world
|
||||
- fix-permissions
|
||||
- csv-to-parquet
|
||||
|
||||
agent:
|
||||
import_path: examples.terminal_bench.praisonai_wrapper_agent:PraisonAIWrapperAgent
|
||||
model_name: openai/gpt-4o-mini
|
||||
env:
|
||||
OPENAI_API_KEY: "${OPENAI_API_KEY}"
|
||||
agents:
|
||||
- import_path: examples.terminal_bench.praisonai_wrapper_agent:PraisonAIWrapperAgent
|
||||
model_name: openai/gpt-4o-mini
|
||||
env:
|
||||
OPENAI_API_KEY: "${OPENAI_API_KEY}"
|
||||
|
||||
n_concurrent: 1
|
||||
n_concurrent_trials: 1
|
||||
n_attempts: 1
|
||||
|
||||
# Filter to 5 specific simple tasks
|
||||
task_filter:
|
||||
task_names:
|
||||
- "hello-world"
|
||||
- "fix-permissions"
|
||||
- "create-bucket"
|
||||
- "csv-to-parquet"
|
||||
- "simple-web-scraper"
|
||||
@@ -13,7 +13,7 @@ The multi-agent approach is useful for Terminal-Bench tasks that benefit from:
|
||||
Usage:
|
||||
harbor run -d terminal-bench/terminal-bench-2 \
|
||||
--agent-import-path examples.terminal_bench.multi_agent_example:MultiAgentPraisonAI \
|
||||
--model openai/gpt-4o
|
||||
--model openai/gpt-4o-mini
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -79,7 +79,7 @@ class MultiAgentPraisonAI(BaseAgent):
|
||||
return "Error: Empty command"
|
||||
|
||||
try:
|
||||
result = await environment.exec(command=command, timeout_sec=30)
|
||||
result = await environment.exec(command=command, timeout_sec=300)
|
||||
output_parts = []
|
||||
if result.stdout:
|
||||
output_parts.append(result.stdout.strip())
|
||||
@@ -101,7 +101,7 @@ class MultiAgentPraisonAI(BaseAgent):
|
||||
"Consider potential issues and edge cases. "
|
||||
"Output your plan as a numbered list of specific actions."
|
||||
),
|
||||
llm=self.model_name or "openai/gpt-4o",
|
||||
llm=self.model_name or "openai/gpt-4o-mini",
|
||||
)
|
||||
|
||||
executor = Agent(
|
||||
@@ -114,7 +114,7 @@ class MultiAgentPraisonAI(BaseAgent):
|
||||
"If a step fails, try alternative approaches."
|
||||
),
|
||||
tools=[bash_tool],
|
||||
llm=self.model_name or "openai/gpt-4o",
|
||||
llm=self.model_name or "openai/gpt-4o-mini",
|
||||
)
|
||||
|
||||
verifier = Agent(
|
||||
@@ -127,7 +127,7 @@ class MultiAgentPraisonAI(BaseAgent):
|
||||
"If issues are found, suggest specific fixes."
|
||||
),
|
||||
tools=[bash_tool],
|
||||
llm=self.model_name or "openai/gpt-4o",
|
||||
llm=self.model_name or "openai/gpt-4o-mini",
|
||||
)
|
||||
|
||||
print(f"🚀 Multi-Agent PraisonAI starting: {instruction[:100]}...")
|
||||
@@ -194,7 +194,7 @@ class MultiAgentPraisonAI(BaseAgent):
|
||||
"framework": "praisonai-multi",
|
||||
"agent_type": "multi-agent-team",
|
||||
"agents": [agent.name for agent in agents],
|
||||
"model": self.model_name or "openai/gpt-4o",
|
||||
"model": self.model_name or "openai/gpt-4o-mini",
|
||||
"phases": ["planning", "execution", "verification"],
|
||||
"tools_used": ["bash_tool"],
|
||||
"result_summary": str(result.get("verification", ""))[:200],
|
||||
@@ -238,7 +238,7 @@ class AgentTeamPraisonAI(BaseAgent):
|
||||
try:
|
||||
# Create bash tool
|
||||
async def bash_tool(command: str) -> str:
|
||||
result = await environment.exec(command=command, timeout_sec=30)
|
||||
result = await environment.exec(command=command, timeout_sec=300)
|
||||
output_parts = []
|
||||
if result.stdout:
|
||||
output_parts.append(result.stdout.strip())
|
||||
@@ -252,14 +252,14 @@ class AgentTeamPraisonAI(BaseAgent):
|
||||
planner = Agent(
|
||||
name="planner",
|
||||
instructions="Create detailed execution plans for terminal tasks",
|
||||
llm=self.model_name or "openai/gpt-4o"
|
||||
llm=self.model_name or "openai/gpt-4o-mini"
|
||||
)
|
||||
|
||||
executor = Agent(
|
||||
name="executor",
|
||||
instructions="Execute terminal commands based on plans",
|
||||
tools=[bash_tool],
|
||||
llm=self.model_name or "openai/gpt-4o"
|
||||
llm=self.model_name or "openai/gpt-4o-mini"
|
||||
)
|
||||
|
||||
# Create tasks
|
||||
@@ -317,12 +317,12 @@ if __name__ == "__main__":
|
||||
print("# Multi-agent custom workflow")
|
||||
print("harbor run -d terminal-bench/terminal-bench-2 \\")
|
||||
print(" --agent-import-path examples.terminal_bench.multi_agent_example:MultiAgentPraisonAI \\")
|
||||
print(" --model openai/gpt-4o")
|
||||
print(" --model openai/gpt-4o-mini")
|
||||
print()
|
||||
print("# AgentTeam structured workflow")
|
||||
print("harbor run -d terminal-bench/terminal-bench-2 \\")
|
||||
print(" --agent-import-path examples.terminal_bench.multi_agent_example:AgentTeamPraisonAI \\")
|
||||
print(" --model openai/gpt-4o")
|
||||
print(" --model openai/gpt-4o-mini")
|
||||
print()
|
||||
print("Benefits of multi-agent approach:")
|
||||
print("- Task decomposition and planning")
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
"""
|
||||
PraisonAI Code Agent for Terminal-Bench 2.1 (Harbor)
|
||||
|
||||
The headline adapter: benchmarks the terminal-native `praisonai code` assistant
|
||||
by installing it inside the Harbor container and driving it headlessly.
|
||||
|
||||
Usage:
|
||||
harbor run -d terminal-bench/terminal-bench-2-1 \
|
||||
--agent "examples.terminal_bench.praisonai_code_agent:PraisonAICodeAgent" \
|
||||
-m openai/gpt-4o-mini \
|
||||
--ae OPENAI_API_KEY=$OPENAI_API_KEY \
|
||||
-n 4
|
||||
|
||||
Architecture:
|
||||
Harbor Container → `praisonai code "TASK" --dangerously-skip-approval` (headless)
|
||||
|
||||
Notes:
|
||||
- `--dangerously-skip-approval` sets PRAISON_APPROVAL_MODE=auto +
|
||||
PRAISONAI_TOOL_SAFETY=off so the assistant runs fully autonomously in the
|
||||
container (no approval hang in a non-TTY session).
|
||||
- `praisonai code` normally exits 0 and Harbor grades by task verification,
|
||||
so a benchmark miss does not fail the run — but the real exit status is
|
||||
propagated so genuine install/auth/startup failures still surface.
|
||||
- The base `praisonai` package is sufficient; heavy `code` extras are not
|
||||
required (ACP tools degrade gracefully).
|
||||
|
||||
Dependencies:
|
||||
pip install harbor praisonai praisonaiagents
|
||||
"""
|
||||
|
||||
import shlex
|
||||
|
||||
try:
|
||||
from harbor.agents.installed.base import BaseInstalledAgent
|
||||
from harbor.environments.base import BaseEnvironment
|
||||
from harbor.models.agent.context import AgentContext
|
||||
except ImportError as e: # pragma: no cover - only importable when Harbor present
|
||||
raise ImportError(
|
||||
f"Harbor framework not installed: {e}\n"
|
||||
"Install with: pip install harbor"
|
||||
) from e
|
||||
|
||||
|
||||
class PraisonAICodeAgent(BaseInstalledAgent):
|
||||
"""Benchmarks the `praisonai code` terminal assistant inside a Harbor container."""
|
||||
|
||||
@staticmethod
|
||||
def name() -> str:
|
||||
return "praisonai-code"
|
||||
|
||||
def get_version_command(self) -> str:
|
||||
return "praisonai --version"
|
||||
|
||||
async def install(self, environment: BaseEnvironment) -> None:
|
||||
"""Install python + the praisonai CLI inside the container."""
|
||||
# System packages (best-effort; image may already have them).
|
||||
await self.exec_as_root(
|
||||
environment,
|
||||
command="apt-get update && apt-get install -y python3 python3-pip || true",
|
||||
env={"DEBIAN_FRONTEND": "noninteractive"},
|
||||
)
|
||||
|
||||
# PEP 668-safe install on bookworm-based images.
|
||||
version_spec = f"=={self._version}" if getattr(self, "_version", None) else ""
|
||||
await self.exec_as_agent(
|
||||
environment,
|
||||
command=(
|
||||
f"pip install --break-system-packages praisonai{version_spec} "
|
||||
"praisonaiagents"
|
||||
),
|
||||
)
|
||||
|
||||
async def run(
|
||||
self,
|
||||
instruction: str,
|
||||
environment: BaseEnvironment,
|
||||
context: AgentContext,
|
||||
) -> None:
|
||||
"""Drive `praisonai code` headlessly on the instruction."""
|
||||
model = self.model_name or "openai/gpt-4o-mini"
|
||||
|
||||
command = (
|
||||
f"praisonai code {shlex.quote(instruction)} "
|
||||
f"--dangerously-skip-approval "
|
||||
f"--model {shlex.quote(model)} "
|
||||
"> /tmp/praisonai_code.log 2>&1; "
|
||||
# Capture the real exit status so install/auth/startup failures still
|
||||
# surface instead of being silently masked (Harbor otherwise records a
|
||||
# completed attempt for a crashed agent).
|
||||
"status=$?; "
|
||||
# Tee the log into a stable path so populate_context_post_run can read it.
|
||||
"cp /tmp/praisonai_code.log /tmp/praisonai_code_run.log 2>/dev/null || true; "
|
||||
# `praisonai code` normally exits 0 (Harbor grades by task
|
||||
# verification), so a genuine benchmark miss won't fail here — but a
|
||||
# nonzero status means the assistant itself failed to run.
|
||||
"exit $status"
|
||||
)
|
||||
|
||||
# `extra_env` is the --ae channel; merge it into the exec env (never host
|
||||
# os.environ). `env` on BaseInstalledAgent already includes extra_env.
|
||||
await self.exec_as_agent(
|
||||
environment,
|
||||
command=command,
|
||||
env=self.env,
|
||||
)
|
||||
|
||||
def populate_context_post_run(self, context: AgentContext) -> None:
|
||||
"""`praisonai code` exposes no machine-readable metrics headlessly.
|
||||
|
||||
Record identifying metadata; token/cost accounting is not available from
|
||||
the CLI's mixed stdout, so we leave those fields unset rather than guess.
|
||||
"""
|
||||
context.metadata = {
|
||||
"framework": "praisonai",
|
||||
"agent_type": "code-cli",
|
||||
"agent_name": self.name(),
|
||||
"model": self.model_name,
|
||||
"log_path": "/tmp/praisonai_code.log",
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("PraisonAI Code Agent for Terminal-Bench 2.1")
|
||||
print("Usage:")
|
||||
print(" harbor run -d terminal-bench/terminal-bench-2-1 \\")
|
||||
print(' --agent "examples.terminal_bench.praisonai_code_agent:PraisonAICodeAgent" \\')
|
||||
print(" -m openai/gpt-4o-mini --ae OPENAI_API_KEY=$OPENAI_API_KEY -n 4")
|
||||
@@ -7,7 +7,7 @@ BaseEnvironment interface. The agent uses Harbor's exec() as a bash tool.
|
||||
Usage:
|
||||
harbor run -d terminal-bench/terminal-bench-2 \
|
||||
--agent-import-path examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent \
|
||||
--model openai/gpt-4o \
|
||||
--model openai/gpt-4o-mini \
|
||||
--ae OPENAI_API_KEY=$OPENAI_API_KEY
|
||||
|
||||
Architecture:
|
||||
@@ -75,8 +75,9 @@ class PraisonAIExternalAgent(BaseAgent):
|
||||
"""
|
||||
|
||||
# Inject API keys from Harbor's --ae env vars into host os.environ
|
||||
# so litellm can pick them up (--ae only sets them inside Docker, not the host)
|
||||
agent_env = getattr(context, 'env', {}) or {}
|
||||
# so litellm can pick them up. The --ae channel arrives via the
|
||||
# BaseAgent.extra_env property (AgentContext has no `env` field).
|
||||
agent_env = getattr(self, 'extra_env', {}) or {}
|
||||
for key, val in agent_env.items():
|
||||
if key not in os.environ and val:
|
||||
os.environ[key] = val
|
||||
@@ -139,7 +140,7 @@ class PraisonAIExternalAgent(BaseAgent):
|
||||
" Keep calling bash_tool until the test PASSES or you have exhausted all approaches."
|
||||
),
|
||||
tools=[bash_tool],
|
||||
llm=self.model_name or "openai/gpt-4o",
|
||||
llm=self.model_name or "openai/gpt-4o-mini",
|
||||
)
|
||||
|
||||
# Execute the agent with outer loop to handle premature stopping
|
||||
@@ -182,17 +183,14 @@ class PraisonAIExternalAgent(BaseAgent):
|
||||
Harbor tracks: n_input_tokens, n_output_tokens, cost_usd, metadata
|
||||
"""
|
||||
try:
|
||||
# Extract token usage and cost from agent
|
||||
# Extract token usage and cost from agent. `cost_summary` is a
|
||||
# property returning a dict (not callable) on praisonaiagents.Agent.
|
||||
try:
|
||||
summary = agent.cost_summary() if callable(getattr(agent, 'cost_summary', None)) else None
|
||||
summary = getattr(agent, 'cost_summary', None)
|
||||
if isinstance(summary, dict):
|
||||
context.n_input_tokens = summary.get('tokens_in')
|
||||
context.n_output_tokens = summary.get('tokens_out')
|
||||
context.cost_usd = summary.get('cost')
|
||||
else:
|
||||
context.n_input_tokens = getattr(agent, '_total_tokens_in', 0)
|
||||
context.n_output_tokens = getattr(agent, '_total_tokens_out', 0)
|
||||
context.cost_usd = getattr(agent, 'total_cost', None)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -216,7 +214,7 @@ if __name__ == "__main__":
|
||||
print("PraisonAI External Agent for Terminal-Bench 2.0")
|
||||
print("Usage: harbor run -d terminal-bench/terminal-bench-2 \\")
|
||||
print(" --agent-import-path examples.terminal_bench.praisonai_external_agent:PraisonAIExternalAgent \\")
|
||||
print(" --model openai/gpt-4o")
|
||||
print(" --model openai/gpt-4o-mini")
|
||||
print()
|
||||
print("Dependencies:")
|
||||
print(" pip install harbor praisonaiagents")
|
||||
@@ -9,7 +9,7 @@ Usage:
|
||||
# First, this would need to be integrated into Harbor's codebase as:
|
||||
# src/harbor/agents/installed/praisonai.py
|
||||
|
||||
harbor run -d terminal-bench/terminal-bench-2 -a praisonai --model openai/gpt-4o
|
||||
harbor run -d terminal-bench/terminal-bench-2 -a praisonai --model openai/gpt-4o-mini
|
||||
|
||||
Architecture:
|
||||
Harbor Container → praisonaiagents installed inside → execute_command tool
|
||||
@@ -25,34 +25,9 @@ import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
try:
|
||||
# These imports would work if this file was in Harbor's codebase
|
||||
from harbor.agents.installed.base import BaseInstalledAgent, with_prompt_template, CliFlag
|
||||
from harbor.environments.base import BaseEnvironment
|
||||
from harbor.models.agent.context import AgentContext
|
||||
from harbor.models.trial.paths import EnvironmentPaths
|
||||
except ImportError:
|
||||
# Fallback for development/testing outside Harbor
|
||||
print("Note: This is a reference implementation for Harbor integration")
|
||||
print("To actually use this, it needs to be integrated into Harbor's codebase")
|
||||
|
||||
# Mock classes for development
|
||||
class BaseInstalledAgent:
|
||||
pass
|
||||
|
||||
def with_prompt_template(func):
|
||||
return func
|
||||
|
||||
class CliFlag:
|
||||
def __init__(self, name, cli, type, default):
|
||||
pass
|
||||
|
||||
class BaseEnvironment:
|
||||
async def exec(self, **kwargs):
|
||||
pass
|
||||
|
||||
class AgentContext:
|
||||
pass
|
||||
from harbor.agents.installed.base import BaseInstalledAgent, with_prompt_template, CliFlag
|
||||
from harbor.environments.base import BaseEnvironment
|
||||
from harbor.models.agent.context import AgentContext
|
||||
|
||||
|
||||
class PraisonAIInstalledAgent(BaseInstalledAgent):
|
||||
@@ -66,12 +41,10 @@ class PraisonAIInstalledAgent(BaseInstalledAgent):
|
||||
# ATIF trajectory support - would be implemented once PraisonAI has trajectory format
|
||||
SUPPORTS_ATIF: bool = False
|
||||
|
||||
# Command-line configuration flags
|
||||
# Command-line configuration flags. The real Harbor CliFlag dataclass keys
|
||||
# the value by `kwarg` (not `name`).
|
||||
CLI_FLAGS = [
|
||||
CliFlag("max_turns", cli="--max-turns", type="int", default=30),
|
||||
CliFlag("verbose", cli="--verbose", type="bool", default=False),
|
||||
CliFlag("memory", cli="--memory", type="bool", default=False),
|
||||
CliFlag("auto_approval", cli="--auto-approval", type="bool", default=True),
|
||||
CliFlag(kwarg="max_turns", cli="--max-turns", type="int", default=30),
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
@@ -142,7 +115,7 @@ def main():
|
||||
sys.exit(1)
|
||||
|
||||
instruction = sys.argv[1]
|
||||
model = sys.argv[2] if len(sys.argv) > 2 else "openai/gpt-4o"
|
||||
model = sys.argv[2] if len(sys.argv) > 2 else "openai/gpt-4o-mini"
|
||||
max_turns = int(sys.argv[3]) if len(sys.argv) > 3 else 30
|
||||
try:
|
||||
from praisonaiagents import Agent
|
||||
@@ -176,19 +149,19 @@ def main():
|
||||
"tools_used": ["execute_command"],
|
||||
}
|
||||
|
||||
# Add token usage if available
|
||||
if hasattr(agent, '_usage'):
|
||||
usage = agent._usage
|
||||
if usage:
|
||||
metrics.update({
|
||||
"input_tokens": getattr(usage, 'input_tokens', None),
|
||||
"output_tokens": getattr(usage, 'output_tokens', None),
|
||||
})
|
||||
|
||||
# Add cost if available
|
||||
if hasattr(agent, '_cost'):
|
||||
metrics["cost_usd"] = agent._cost
|
||||
|
||||
# Token usage + cost come from the `cost_summary` property (a dict).
|
||||
try:
|
||||
summary = getattr(agent, "cost_summary", None)
|
||||
if isinstance(summary, dict):
|
||||
metrics["input_tokens"] = summary.get("tokens_in")
|
||||
metrics["output_tokens"] = summary.get("tokens_out")
|
||||
metrics["cost_usd"] = summary.get("cost")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Write metrics to a stable path so the host adapter can read them back.
|
||||
with open("/tmp/praisonai_metrics.json", "w") as fh:
|
||||
json.dump(metrics, fh)
|
||||
print(json.dumps(metrics))
|
||||
|
||||
except Exception as e:
|
||||
@@ -216,7 +189,7 @@ if __name__ == "__main__":
|
||||
This executes the headless runner script with the instruction,
|
||||
similar to how other installed agents work in Harbor.
|
||||
"""
|
||||
model = self.model_name or "openai/gpt-4o"
|
||||
model = self.model_name or "openai/gpt-4o-mini"
|
||||
max_turns = getattr(self, 'max_turns', 30)
|
||||
|
||||
# Build command to run PraisonAI
|
||||
@@ -227,15 +200,18 @@ if __name__ == "__main__":
|
||||
str(max_turns),
|
||||
]
|
||||
|
||||
command = " ".join(cmd_args)
|
||||
# Tee the metrics file back so populate_context_post_run can read it via
|
||||
# the container. Also print it so it lands in Harbor's captured stdout.
|
||||
command = " ".join(cmd_args) + "; cat /tmp/praisonai_metrics.json 2>/dev/null || true"
|
||||
|
||||
try:
|
||||
# Execute the agent
|
||||
await self.exec_as_agent(
|
||||
# Execute the agent and keep the ExecResult stdout for metric parsing.
|
||||
result = await self.exec_as_agent(
|
||||
environment,
|
||||
command=command,
|
||||
env=self._get_environment_vars(),
|
||||
env=self.env,
|
||||
)
|
||||
self._last_stdout = getattr(result, "stdout", "") or ""
|
||||
except Exception as e:
|
||||
# Store error in context for Harbor's reporting
|
||||
context.metadata = {"execution_error": str(e)}
|
||||
@@ -270,37 +246,38 @@ if __name__ == "__main__":
|
||||
JSON output of the headless runner script.
|
||||
"""
|
||||
try:
|
||||
# Parse the last stdout output for JSON metrics
|
||||
# In Harbor's model, the last execution output should contain our JSON
|
||||
last_output = getattr(context, '_last_stdout', None)
|
||||
|
||||
# Parse the JSON metrics captured from the runner's stdout (teed via
|
||||
# `cat /tmp/praisonai_metrics.json` in run()). AgentContext has no
|
||||
# `_last_stdout`; we stash it on the agent instead.
|
||||
last_output = getattr(self, '_last_stdout', None)
|
||||
metrics = None
|
||||
if last_output:
|
||||
try:
|
||||
metrics = json.loads(last_output.strip())
|
||||
|
||||
# Extract metrics with safe defaults
|
||||
context.n_input_tokens = metrics.get('input_tokens')
|
||||
context.n_output_tokens = metrics.get('output_tokens')
|
||||
context.cost_usd = metrics.get('cost_usd')
|
||||
|
||||
# Store additional metadata
|
||||
context.metadata = {
|
||||
"framework": "praisonai",
|
||||
"agent_type": "installed",
|
||||
"agent_name": metrics.get('agent_name', 'terminal-agent'),
|
||||
"model": metrics.get('model'),
|
||||
"tools_used": metrics.get('tools_used', []),
|
||||
"version": self.get_version() if hasattr(self, 'get_version') else None,
|
||||
}
|
||||
|
||||
except (json.JSONDecodeError, ValueError) as e:
|
||||
# If JSON parsing fails, store basic metadata
|
||||
context.metadata = {
|
||||
"framework": "praisonai",
|
||||
"agent_type": "installed",
|
||||
"parse_error": str(e),
|
||||
"raw_output": str(last_output)[:200] if last_output else None,
|
||||
}
|
||||
# The output may contain CLI noise before the JSON line; grab the
|
||||
# last line that parses as a JSON object.
|
||||
for line in reversed(last_output.strip().splitlines()):
|
||||
line = line.strip()
|
||||
if line.startswith("{"):
|
||||
try:
|
||||
metrics = json.loads(line)
|
||||
break
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
continue
|
||||
|
||||
if metrics is not None:
|
||||
# Extract metrics with safe defaults
|
||||
context.n_input_tokens = metrics.get('input_tokens')
|
||||
context.n_output_tokens = metrics.get('output_tokens')
|
||||
context.cost_usd = metrics.get('cost_usd')
|
||||
|
||||
# Store additional metadata
|
||||
context.metadata = {
|
||||
"framework": "praisonai",
|
||||
"agent_type": "installed",
|
||||
"agent_name": metrics.get('agent_name', 'terminal-agent'),
|
||||
"model": metrics.get('model'),
|
||||
"tools_used": metrics.get('tools_used', []),
|
||||
"version": self.get_version() if hasattr(self, 'get_version') else None,
|
||||
}
|
||||
else:
|
||||
# No output to parse
|
||||
context.metadata = {
|
||||
@@ -334,7 +311,7 @@ if __name__ == "__main__":
|
||||
print(" - Add PraisonAI to _AGENTS list and _AGENT_MAP")
|
||||
print()
|
||||
print("3. Run with Harbor:")
|
||||
print(" harbor run -d terminal-bench/terminal-bench-2 -a praisonai --model openai/gpt-4o")
|
||||
print(" harbor run -d terminal-bench/terminal-bench-2 -a praisonai --model openai/gpt-4o-mini")
|
||||
print()
|
||||
print("Dependencies:")
|
||||
print(" pip install harbor praisonaiagents")
|
||||
@@ -8,7 +8,7 @@ that matches the standard `praisonai "TASK"` CLI usage pattern.
|
||||
Usage:
|
||||
harbor run -d terminal-bench/terminal-bench-2 \
|
||||
--agent-import-path examples.terminal_bench.praisonai_wrapper_agent:PraisonAIWrapperAgent \
|
||||
--model openai/gpt-4o \
|
||||
--model openai/gpt-4o-mini \
|
||||
--ae OPENAI_API_KEY=$OPENAI_API_KEY \
|
||||
-n 4
|
||||
|
||||
@@ -95,13 +95,16 @@ class PraisonAIWrapperAgent(BaseAgent):
|
||||
|
||||
Uses the `praisonai "TASK"` CLI pattern inside the Harbor container.
|
||||
"""
|
||||
model = self.model_name or "openai/gpt-4o"
|
||||
model = self.model_name or "openai/gpt-4o-mini"
|
||||
|
||||
# Build the praisonai CLI command
|
||||
# Format: praisonai "TASK" --model MODEL
|
||||
# Build the `praisonai code` headless command. --dangerously-skip-approval
|
||||
# makes the terminal assistant run autonomously (no approval hang in a
|
||||
# non-TTY container). Bare `praisonai "TASK"` would run agents-generator
|
||||
# mode with no shell-tool loop, so we use the code assistant instead.
|
||||
cmd_parts = [
|
||||
"praisonai",
|
||||
"praisonai", "code",
|
||||
shlex.quote(instruction),
|
||||
"--dangerously-skip-approval",
|
||||
"--model", shlex.quote(model),
|
||||
]
|
||||
|
||||
@@ -148,7 +151,7 @@ class PraisonAIWrapperAgent(BaseAgent):
|
||||
# Store result summary
|
||||
context.metadata = {
|
||||
"agent_name": "praisonai-wrapper",
|
||||
"model": self.model_name or "openai/gpt-4o",
|
||||
"model": self.model_name or "openai/gpt-4o-mini",
|
||||
"framework": "praisonai",
|
||||
"wrapper_type": "cli",
|
||||
"instruction_preview": instruction[:200],
|
||||
@@ -204,16 +207,16 @@ class PraisonAIWrapperAgent(BaseAgent):
|
||||
return env_vars
|
||||
|
||||
async def _exec_as_root(self, environment: BaseEnvironment, command: str, env: Optional[Dict] = None) -> Any:
|
||||
"""Execute command as root in the container."""
|
||||
if hasattr(environment, 'exec_as_root'):
|
||||
return await environment.exec_as_root(command=command, env=env or {})
|
||||
else:
|
||||
# Fallback: use sudo
|
||||
return await environment.exec(command=f"sudo {command}", env=env or {})
|
||||
"""Execute command as root in the container.
|
||||
|
||||
async def _exec_as_agent(self, environment: BaseEnvironment, command: str) -> Any:
|
||||
BaseEnvironment has no `exec_as_root`; use `exec(..., user="root")`.
|
||||
`sudo` is typically absent from TB task containers.
|
||||
"""
|
||||
return await environment.exec(command=command, env=env or {}, user="root")
|
||||
|
||||
async def _exec_as_agent(self, environment: BaseEnvironment, command: str, env: Optional[Dict] = None) -> Any:
|
||||
"""Execute command as the agent user in the container."""
|
||||
return await environment.exec(command=command)
|
||||
return await environment.exec(command=command, env=env or {})
|
||||
|
||||
|
||||
# Example usage for testing
|
||||
@@ -227,7 +230,7 @@ if __name__ == "__main__":
|
||||
print("Usage with Harbor:")
|
||||
print(" harbor run -d terminal-bench/terminal-bench-2 \\")
|
||||
print(" --agent-import-path examples.terminal_bench.praisonai_wrapper_agent:PraisonAIWrapperAgent \\")
|
||||
print(" --model openai/gpt-4o")
|
||||
print(" --model openai/gpt-4o-mini")
|
||||
print()
|
||||
print("Dependencies:")
|
||||
print(" pip install harbor praisonai")
|
||||
@@ -1,274 +1,119 @@
|
||||
"""
|
||||
Tests for PraisonAI Terminal-Bench 2.0 Integration
|
||||
Tests for the PraisonAI Terminal-Bench 2.1 (Harbor) integration.
|
||||
|
||||
This module contains both unit tests and integration tests for the
|
||||
PraisonAI Harbor integration.
|
||||
Run from the repo root with PYTHONPATH set so the package import path resolves:
|
||||
PYTHONPATH=. python -m pytest examples/terminal_bench/test_integration.py -v
|
||||
|
||||
Run tests with:
|
||||
python -m pytest examples/terminal_bench/test_integration.py -v
|
||||
|
||||
Requirements:
|
||||
pip install pytest praisonaiagents
|
||||
# Harbor is optional for unit tests, required for integration tests
|
||||
Tests that require Harbor are honestly skipped only when Harbor is not installed
|
||||
(checked via importlib.util.find_spec). pytest-asyncio must be installed for the
|
||||
async tests.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import shlex
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
import asyncio
|
||||
from unittest.mock import Mock, AsyncMock, patch
|
||||
from pathlib import Path
|
||||
|
||||
HARBOR_AVAILABLE = importlib.util.find_spec("harbor") is not None
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not HARBOR_AVAILABLE, reason="Harbor not installed"
|
||||
)
|
||||
|
||||
|
||||
class TestPraisonAICodeAgent:
|
||||
"""The headline `praisonai code` adapter."""
|
||||
|
||||
def test_metadata(self):
|
||||
from examples.terminal_bench.praisonai_code_agent import PraisonAICodeAgent
|
||||
|
||||
agent = PraisonAICodeAgent(model_name="openai/gpt-4o-mini", logs_dir="/tmp")
|
||||
assert agent.name() == "praisonai-code"
|
||||
assert agent.get_version_command() == "praisonai --version"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_code_agent_command_shape(self, tmp_path):
|
||||
"""run() must issue the exact headless `praisonai code` command."""
|
||||
from examples.terminal_bench.praisonai_code_agent import PraisonAICodeAgent
|
||||
|
||||
agent = PraisonAICodeAgent(
|
||||
model_name="openai/gpt-4o-mini", logs_dir=str(tmp_path)
|
||||
)
|
||||
env = Mock()
|
||||
env.exec = AsyncMock(
|
||||
return_value=SimpleNamespace(stdout="", stderr="", return_code=0)
|
||||
)
|
||||
context = SimpleNamespace(metadata=None)
|
||||
|
||||
instruction = "fix the failing test"
|
||||
await agent.run(instruction, env, context)
|
||||
|
||||
env.exec.assert_awaited()
|
||||
command = env.exec.await_args.kwargs.get("command", "")
|
||||
assert command.startswith(f"praisonai code {shlex.quote(instruction)}")
|
||||
assert "--dangerously-skip-approval" in command
|
||||
assert "--model openai/gpt-4o-mini" in command
|
||||
|
||||
|
||||
class TestPraisonAIExternalAgent:
|
||||
"""Test the external agent implementation."""
|
||||
|
||||
def test_agent_metadata(self):
|
||||
"""Test agent name and version reporting."""
|
||||
try:
|
||||
from .praisonai_external_agent import PraisonAIExternalAgent
|
||||
except ImportError:
|
||||
pytest.skip("Harbor not installed - skipping Harbor-specific tests")
|
||||
|
||||
"""The external (SDK-level) adapter."""
|
||||
|
||||
def test_metadata(self):
|
||||
from examples.terminal_bench.praisonai_external_agent import (
|
||||
PraisonAIExternalAgent,
|
||||
)
|
||||
|
||||
agent = PraisonAIExternalAgent()
|
||||
assert agent.name() == "praisonai"
|
||||
|
||||
# Version should be None if praisonaiagents not installed, or actual version
|
||||
version = agent.version()
|
||||
assert version is None or isinstance(version, str)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_setup(self):
|
||||
"""Test agent setup phase."""
|
||||
try:
|
||||
from .praisonai_external_agent import PraisonAIExternalAgent
|
||||
except ImportError:
|
||||
pytest.skip("Harbor not installed")
|
||||
|
||||
agent = PraisonAIExternalAgent()
|
||||
mock_env = Mock()
|
||||
|
||||
# Setup should complete without error (external agent needs no setup)
|
||||
await agent.setup(mock_env)
|
||||
def test_context_population_uses_cost_summary(self):
|
||||
from examples.terminal_bench.praisonai_external_agent import (
|
||||
PraisonAIExternalAgent,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_tool_execution(self):
|
||||
"""Test the bash tool wrapper around Harbor's exec()."""
|
||||
try:
|
||||
from .praisonai_external_agent import PraisonAIExternalAgent
|
||||
except ImportError:
|
||||
pytest.skip("Harbor not installed")
|
||||
|
||||
# Mock Harbor environment
|
||||
mock_env = Mock()
|
||||
mock_result = Mock()
|
||||
mock_result.stdout = "Hello, World!"
|
||||
mock_result.stderr = ""
|
||||
mock_result.return_code = 0
|
||||
mock_env.exec = AsyncMock(return_value=mock_result)
|
||||
|
||||
# Mock agent context
|
||||
mock_context = Mock()
|
||||
mock_context.metadata = {}
|
||||
|
||||
agent = PraisonAIExternalAgent()
|
||||
|
||||
# Mock PraisonAI Agent to avoid LLM calls in tests
|
||||
with patch('praisonaiagents.Agent') as mock_agent_class:
|
||||
mock_agent_instance = Mock()
|
||||
mock_agent_instance.start.return_value = "Task completed successfully"
|
||||
mock_agent_instance.name = "terminal-agent"
|
||||
mock_agent_class.return_value = mock_agent_instance
|
||||
|
||||
# Mock approval backend
|
||||
with patch('praisonaiagents.approval.set_approval_backend'):
|
||||
await agent.run("echo 'Hello, World!'", mock_env, mock_context)
|
||||
|
||||
# Verify agent was created with correct parameters
|
||||
mock_agent_class.assert_called_once()
|
||||
args, kwargs = mock_agent_class.call_args
|
||||
|
||||
assert kwargs['name'] == 'terminal-agent'
|
||||
assert 'tools' in kwargs
|
||||
assert len(kwargs['tools']) == 1 # bash_tool
|
||||
assert kwargs['verbose'] is False
|
||||
assert kwargs['memory'] is False
|
||||
|
||||
def test_context_population(self):
|
||||
"""Test that agent context is properly populated."""
|
||||
try:
|
||||
from .praisonai_external_agent import PraisonAIExternalAgent
|
||||
except ImportError:
|
||||
pytest.skip("Harbor not installed")
|
||||
|
||||
agent_impl = PraisonAIExternalAgent()
|
||||
|
||||
# Mock agent with usage data
|
||||
|
||||
# cost_summary is a dict property on the real Agent, not a callable.
|
||||
mock_agent = Mock()
|
||||
mock_agent.name = "test-agent"
|
||||
mock_agent.llm = "gpt-4o"
|
||||
mock_agent._usage = Mock()
|
||||
mock_agent._usage.input_tokens = 100
|
||||
mock_agent._usage.output_tokens = 50
|
||||
mock_agent._cost = 0.01
|
||||
|
||||
mock_context = Mock()
|
||||
mock_context.metadata = {}
|
||||
|
||||
# Test context population
|
||||
agent_impl._populate_context(mock_agent, mock_context, "Test result")
|
||||
|
||||
assert mock_context.n_input_tokens == 100
|
||||
assert mock_context.n_output_tokens == 50
|
||||
assert mock_context.cost_usd == 0.01
|
||||
assert mock_context.metadata['agent_name'] == 'test-agent'
|
||||
assert mock_context.metadata['model'] == 'gpt-4o'
|
||||
assert mock_context.metadata['framework'] == 'praisonai'
|
||||
mock_agent.llm = "gpt-4o-mini"
|
||||
mock_agent.cost_summary = {"tokens_in": 100, "tokens_out": 50, "cost": 0.01}
|
||||
|
||||
context = SimpleNamespace(
|
||||
n_input_tokens=None, n_output_tokens=None, cost_usd=None, metadata=None
|
||||
)
|
||||
agent_impl._populate_context(mock_agent, context, "done")
|
||||
|
||||
assert context.n_input_tokens == 100
|
||||
assert context.n_output_tokens == 50
|
||||
assert context.cost_usd == 0.01
|
||||
assert context.metadata["framework"] == "praisonai"
|
||||
|
||||
|
||||
class TestPraisonAIInstalledAgent:
|
||||
"""Test the installed agent implementation."""
|
||||
|
||||
def test_agent_configuration(self):
|
||||
"""Test agent CLI flags and configuration."""
|
||||
try:
|
||||
from .praisonai_installed_agent import PraisonAIInstalledAgent
|
||||
except ImportError:
|
||||
pytest.skip("Harbor not installed")
|
||||
|
||||
agent = PraisonAIInstalledAgent()
|
||||
assert agent.name() == "praisonai"
|
||||
assert agent.SUPPORTS_ATIF is False # Until trajectory format is implemented
|
||||
|
||||
# Check CLI flags
|
||||
flag_names = [flag.name if hasattr(flag, 'name') else str(flag) for flag in agent.CLI_FLAGS]
|
||||
expected_flags = ['max_turns', 'verbose', 'memory', 'auto_approval']
|
||||
for expected in expected_flags:
|
||||
assert any(expected in flag_name for flag_name in flag_names)
|
||||
"""The installed adapter."""
|
||||
|
||||
def test_version_command(self):
|
||||
"""Test version detection command."""
|
||||
try:
|
||||
from .praisonai_installed_agent import PraisonAIInstalledAgent
|
||||
except ImportError:
|
||||
pytest.skip("Harbor not installed")
|
||||
|
||||
agent = PraisonAIInstalledAgent()
|
||||
version_cmd = agent.get_version_command()
|
||||
|
||||
assert version_cmd is not None
|
||||
assert "praisonaiagents" in version_cmd
|
||||
assert "__version__" in version_cmd
|
||||
|
||||
def test_runner_script_generation(self):
|
||||
"""Test that the headless runner script is properly generated."""
|
||||
try:
|
||||
from .praisonai_installed_agent import PraisonAIInstalledAgent
|
||||
except ImportError:
|
||||
pytest.skip("Harbor not installed")
|
||||
|
||||
agent = PraisonAIInstalledAgent()
|
||||
script = agent._build_runner_script()
|
||||
|
||||
assert "import praisonaiagents" in script
|
||||
assert "Agent" in script
|
||||
assert "execute_command" in script
|
||||
assert "AutoApproveBackend" in script
|
||||
assert "json.dumps" in script
|
||||
|
||||
|
||||
class TestIntegration:
|
||||
"""Integration tests that require both PraisonAI and Harbor."""
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_praisonai_agent_real(self):
|
||||
"""
|
||||
Real agentic test - agent must call LLM end-to-end.
|
||||
|
||||
This is a MANDATORY test per AGENTS.md §9.4.
|
||||
Agent MUST call agent.start() with real prompt and produce LLM output.
|
||||
"""
|
||||
try:
|
||||
from praisonaiagents import Agent
|
||||
except ImportError:
|
||||
pytest.skip("PraisonAI not installed")
|
||||
|
||||
# Create real agent that will call LLM
|
||||
agent = Agent(
|
||||
name="test-terminal-agent",
|
||||
instructions="You are a helpful terminal assistant"
|
||||
def test_metadata_and_flags(self):
|
||||
from examples.terminal_bench.praisonai_installed_agent import (
|
||||
PraisonAIInstalledAgent,
|
||||
)
|
||||
|
||||
# Real agentic test - agent must call LLM and produce text response
|
||||
result = agent.start("Say hello in one sentence and mention you can help with terminal tasks")
|
||||
|
||||
# Verify we got actual LLM output
|
||||
assert result is not None
|
||||
assert isinstance(result, str)
|
||||
assert len(result) > 0
|
||||
|
||||
# Print output for manual verification
|
||||
print("✅ Real agentic test result:")
|
||||
print(result)
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_tool_real_execution(self):
|
||||
"""Test bash tool with real command execution (if safe)."""
|
||||
# This would test real bash execution in a safe environment
|
||||
# For now, we'll mock it to avoid system changes
|
||||
|
||||
mock_result = Mock()
|
||||
mock_result.stdout = "PraisonAI Terminal Test\n"
|
||||
mock_result.stderr = ""
|
||||
mock_result.return_code = 0
|
||||
|
||||
# Test that our bash tool wrapper works correctly
|
||||
async def mock_exec(command, timeout_sec=30):
|
||||
assert "echo" in command # Ensure we're testing echo command
|
||||
return mock_result
|
||||
|
||||
# This simulates Harbor's BaseEnvironment.exec()
|
||||
result = await mock_exec("echo 'PraisonAI Terminal Test'")
|
||||
|
||||
assert result.stdout.strip() == "PraisonAI Terminal Test"
|
||||
assert result.return_code == 0
|
||||
agent = PraisonAIInstalledAgent(logs_dir="/tmp")
|
||||
assert agent.name() == "praisonai"
|
||||
# CliFlag is keyed by `kwarg` in the real Harbor dataclass.
|
||||
assert all(getattr(f, "kwarg", None) for f in agent.CLI_FLAGS)
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_auto_approval_setup(self):
|
||||
"""Test that auto-approval backend works correctly."""
|
||||
try:
|
||||
from praisonaiagents.approval import set_approval_backend, AutoApproveBackend
|
||||
except ImportError:
|
||||
pytest.skip("PraisonAI approval system not available")
|
||||
|
||||
# Test setting and restoring approval backend
|
||||
original = set_approval_backend(AutoApproveBackend())
|
||||
new_backend = set_approval_backend(original)
|
||||
|
||||
assert isinstance(new_backend, AutoApproveBackend)
|
||||
def test_runner_script_uses_cost_summary(self):
|
||||
from examples.terminal_bench.praisonai_installed_agent import (
|
||||
PraisonAIInstalledAgent,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Allow running tests directly
|
||||
import sys
|
||||
|
||||
print("PraisonAI Terminal-Bench 2.0 Integration Tests")
|
||||
print("=" * 50)
|
||||
|
||||
# Check dependencies
|
||||
try:
|
||||
import praisonaiagents
|
||||
print(f"✅ PraisonAI version: {getattr(praisonaiagents, '__version__', None)}")
|
||||
except ImportError:
|
||||
print("❌ PraisonAI not installed: pip install praisonaiagents")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
import harbor
|
||||
print("✅ Harbor framework available")
|
||||
except ImportError:
|
||||
print("⚠️ Harbor not installed: pip install harbor")
|
||||
print(" (Some tests will be skipped)")
|
||||
|
||||
print()
|
||||
print("Run tests with: python -m pytest examples/terminal_bench/test_integration.py -v")
|
||||
print("Run real agentic test: python -m pytest examples/terminal_bench/test_integration.py::TestIntegration::test_praisonai_agent_real -v -s")
|
||||
agent = PraisonAIInstalledAgent(logs_dir="/tmp")
|
||||
script = agent._build_runner_script()
|
||||
assert "cost_summary" in script
|
||||
assert "_usage" not in script
|
||||
assert "execute_command" in script
|
||||
@@ -13,7 +13,7 @@ Available presets:
|
||||
- json: JSONL output for piping
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent, Workflow
|
||||
from praisonaiagents import Agent, AgentFlow, Workflow
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
@@ -21,20 +21,20 @@ This report compares **Python SDK feature categories** against **Python document
|
||||
| ✅ Agent | 21 | 45 | 14056 |
|
||||
| ✅ Agent-to-Agent (A2A) | 1 | 7 | 2965 |
|
||||
| ✅ Agent-to-User (A2U) | 1 | 3 | 585 |
|
||||
| ✅ Approval | 1 | 5 | 2262 |
|
||||
| ✅ Approval | 1 | 5 | 2286 |
|
||||
| ✅ Audio | 2 | 12 | 1000 |
|
||||
| ✅ Auto Generation | 5 | 10 | 2868 |
|
||||
| ✅ Autonomy | 3 | 7 | 2705 |
|
||||
| ✅ Bots | 7 | 28 | 10942 |
|
||||
| ✅ Autonomy | 3 | 7 | 2707 |
|
||||
| ✅ Bots | 7 | 28 | 11076 |
|
||||
| ✅ Budget | 1 | 1 | 287 |
|
||||
| ✅ CLI | 5 | 116 | 31916 |
|
||||
| ✅ CLI | 5 | 118 | 32713 |
|
||||
| ✅ Chunking | 2 | 2 | 385 |
|
||||
| ✅ Citations | 2 | 1 | 202 |
|
||||
| ✅ Code Execution | 2 | 13 | 3907 |
|
||||
| ✅ Conditions | 1 | 3 | 1179 |
|
||||
| ✅ Configuration | 3 | 7 | 2763 |
|
||||
| ✅ Context Management | 16 | 34 | 13176 |
|
||||
| ✅ Database | 1 | 40 | 6124 |
|
||||
| ✅ Configuration | 3 | 7 | 2772 |
|
||||
| ✅ Context Management | 16 | 34 | 13204 |
|
||||
| ✅ Database | 1 | 40 | 6184 |
|
||||
| ✅ Deep Research | 8 | 2 | 587 |
|
||||
| ✅ Display | 6 | 3 | 741 |
|
||||
| ✅ Embeddings | 6 | 23 | 2049 |
|
||||
@@ -44,45 +44,45 @@ This report compares **Python SDK feature categories** against **Python document
|
||||
| ✅ Failover | 2 | 1 | 400 |
|
||||
| ✅ Files | 2 | 6 | 1964 |
|
||||
| ✅ Flow | 1 | 3 | 777 |
|
||||
| ✅ Gateway | 7 | 49 | 17140 |
|
||||
| ✅ Gateway | 7 | 52 | 18697 |
|
||||
| ✅ Guardrails | 4 | 4 | 1673 |
|
||||
| ✅ Handoffs | 11 | 6 | 2454 |
|
||||
| ✅ Hooks | 2 | 9 | 3928 |
|
||||
| ✅ Hooks | 2 | 9 | 3981 |
|
||||
| ✅ Image | 1 | 11 | 1255 |
|
||||
| ✅ Knowledge | 4 | 13 | 4160 |
|
||||
| ✅ LLM | 3 | 14 | 4767 |
|
||||
| ✅ Loops | 4 | 4 | 1202 |
|
||||
| ✅ MCP | 1 | 58 | 13057 |
|
||||
| ✅ Memory | 6 | 17 | 6481 |
|
||||
| ✅ MCP | 1 | 58 | 13322 |
|
||||
| ✅ Memory | 6 | 17 | 6635 |
|
||||
| ✅ OCR | 2 | 1 | 237 |
|
||||
| ✅ Observability | 2 | 23 | 2771 |
|
||||
| ✅ Observability | 2 | 23 | 2779 |
|
||||
| ✅ Optimizer | 1 | 2 | 792 |
|
||||
| ✅ Output | 3 | 5 | 1158 |
|
||||
| ✅ Parallel Execution | 3 | 2 | 497 |
|
||||
| ✅ Planning | 6 | 6 | 1545 |
|
||||
| ✅ Plugins | 8 | 3 | 1879 |
|
||||
| ✅ Prompts | 2 | 8 | 1726 |
|
||||
| ✅ Providers | 1 | 55 | 7455 |
|
||||
| ✅ Providers | 1 | 55 | 7459 |
|
||||
| ✅ Query | 1 | 2 | 736 |
|
||||
| ✅ RAG | 5 | 15 | 2936 |
|
||||
| ✅ Realtime | 2 | 5 | 668 |
|
||||
| ✅ Reflection | 3 | 3 | 680 |
|
||||
| ✅ Retrieval | 2 | 5 | 1044 |
|
||||
| ✅ Routing | 1 | 2 | 430 |
|
||||
| ✅ Sandbox | 5 | 6 | 2686 |
|
||||
| ✅ Sandbox | 5 | 6 | 2700 |
|
||||
| ✅ Security | 1 | 3 | 2903 |
|
||||
| ✅ Sessions | 4 | 13 | 5198 |
|
||||
| ✅ Skills | 6 | 13 | 4893 |
|
||||
| ✅ Sessions | 4 | 13 | 5286 |
|
||||
| ✅ Skills | 6 | 13 | 4946 |
|
||||
| ✅ Tasks | 2 | 6 | 2743 |
|
||||
| ✅ Telemetry | 1 | 2 | 604 |
|
||||
| ✅ Templates | 1 | 8 | 1657 |
|
||||
| ✅ Tools | 12 | 131 | 37117 |
|
||||
| ✅ Tools | 12 | 132 | 37466 |
|
||||
| ✅ Tracing | 3 | 2 | 139 |
|
||||
| ✅ Vector Store | 1 | 12 | 1193 |
|
||||
| ✅ Video | 2 | 6 | 649 |
|
||||
| ✅ Vision | 2 | 1 | 329 |
|
||||
| ✅ Web | 3 | 8 | 2055 |
|
||||
| ✅ Workflows | 5 | 16 | 6522 |
|
||||
| ✅ Workflows | 5 | 16 | 6527 |
|
||||
|
||||
## Documentation Without Features
|
||||
|
||||
|
||||
@@ -515,8 +515,7 @@ _LAZY_IMPORTS = {
|
||||
'ManagerConfig': ('praisonaiagents.context.manager', 'ManagerConfig'),
|
||||
'ContextManager': ('praisonaiagents.context.manager', 'ContextManager'),
|
||||
|
||||
# db module
|
||||
'db': ('praisonaiagents.db', 'db'),
|
||||
# db shortcut (handled by custom_handler to return _LazyDbModule instance)
|
||||
# Note: 'obs' is handled by custom_handler to return _LazyObsModule instance
|
||||
|
||||
# Gateway protocols and config (implementations in praisonai wrapper)
|
||||
@@ -711,9 +710,15 @@ def _custom_handler(name, cache):
|
||||
return mod.obs
|
||||
if name == 'db':
|
||||
import importlib
|
||||
import sys
|
||||
mod = importlib.import_module('.db', 'praisonaiagents')
|
||||
cache['db'] = mod
|
||||
return mod
|
||||
# importlib registers the submodule as praisonaiagents.db in the package
|
||||
# namespace, so plain attribute lookup would bypass __getattr__ and return
|
||||
# the raw module on repeat access. Rebind the package global to the callable
|
||||
# _LazyDbModule proxy so `praisonaiagents.db(...)` keeps working every time.
|
||||
cache['db'] = mod.db # Return the _LazyDbModule instance, not the module
|
||||
setattr(sys.modules['praisonaiagents'], 'db', mod.db)
|
||||
return mod.db
|
||||
if name == 'toolsets':
|
||||
import importlib
|
||||
mod = importlib.import_module('.toolsets', 'praisonaiagents')
|
||||
|
||||
@@ -1273,16 +1273,42 @@ class Agent(GoalLoopMixin, SteeringMixin, SandboxMixin, SkillReviewMixin, Unifie
|
||||
session_id = _history_session_id
|
||||
elif _history_enabled and session_id is None and _history_session_id is None:
|
||||
import hashlib as _hl
|
||||
_agent_hash = _hl.sha256((name or "agent").encode()).hexdigest()[:8]
|
||||
# Backward compat: check if legacy md5-based session exists first
|
||||
_legacy_hash = _hl.md5((name or "agent").encode()).hexdigest()[:8]
|
||||
_legacy_id = f"history_{_legacy_hash}"
|
||||
_new_id = f"history_{_agent_hash}"
|
||||
# Prefer legacy if it exists on disk, else use new SHA-256 ID
|
||||
import os as _os
|
||||
_session_dir = _os.path.join(_os.path.expanduser("~"), ".praisonai", "sessions")
|
||||
if _os.path.exists(_os.path.join(_session_dir, f"{_legacy_id}.json")):
|
||||
session_id = _legacy_id # preserve existing history
|
||||
_name = name or "agent"
|
||||
# Workspace-scoped id so same-named agents in different projects don't
|
||||
# collide (Issue #3154). Opt out with PRAISONAI_GLOBAL_SESSIONS=true for
|
||||
# name-only global continuity.
|
||||
_global_scope = _os.environ.get("PRAISONAI_GLOBAL_SESSIONS", "").lower() in ("1", "true", "yes")
|
||||
if _global_scope:
|
||||
_workspace_id = "global"
|
||||
else:
|
||||
from ..session.workspace import workspace_id as _wid
|
||||
_workspace_id = _wid()
|
||||
_workspace_hash = _hl.sha256(f"{_workspace_id}:{_name}".encode()).hexdigest()[:8]
|
||||
_new_id = f"history_{_workspace_hash}"
|
||||
# Backward-compat ids (resolution order: name-only sha256, then md5)
|
||||
_name_sha_id = f"history_{_hl.sha256(_name.encode()).hexdigest()[:8]}"
|
||||
_name_md5_id = f"history_{_hl.md5(_name.encode()).hexdigest()[:8]}"
|
||||
# Resolve the migration lookup against the SAME directory the session
|
||||
# store uses at runtime, so a custom PRAISONAI_HOME is honoured (the
|
||||
# store reads from get_sessions_dir()). Fall back to the default path
|
||||
# only if that helper is unavailable.
|
||||
try:
|
||||
from ..paths import get_sessions_dir as _get_sessions_dir
|
||||
_session_dir = str(_get_sessions_dir())
|
||||
except Exception:
|
||||
_session_dir = _os.path.join(_os.path.expanduser("~"), ".praisonai", "sessions")
|
||||
# Prefer an existing workspace-scoped session. Legacy (pre-workspace)
|
||||
# name-only files are only adopted in global scope: doing so in a
|
||||
# workspace-scoped run would silently share Project A's history with a
|
||||
# same-named agent in Project B (Issue #3154). Global scope is the
|
||||
# explicit opt-in for that name-only continuity.
|
||||
if _os.path.exists(_os.path.join(_session_dir, f"{_new_id}.json")):
|
||||
session_id = _new_id
|
||||
elif _global_scope and _os.path.exists(_os.path.join(_session_dir, f"{_name_sha_id}.json")):
|
||||
session_id = _name_sha_id # preserve existing history
|
||||
elif _global_scope and _os.path.exists(_os.path.join(_session_dir, f"{_name_md5_id}.json")):
|
||||
session_id = _name_md5_id # preserve existing history
|
||||
else:
|
||||
session_id = _new_id
|
||||
_history_session_id = session_id
|
||||
@@ -1759,6 +1785,7 @@ class Agent(GoalLoopMixin, SteeringMixin, SandboxMixin, SkillReviewMixin, Unifie
|
||||
if toolsets:
|
||||
try:
|
||||
from ..toolsets import resolve_toolsets_for_model
|
||||
from ..tools.resolver import ToolResolutionError
|
||||
# Advertise the model-family's preferred edit primitive first
|
||||
# (e.g. apply_patch for Claude, edit_file for GPT). Unknown /
|
||||
# non-string models fall back to the byte-for-byte default order.
|
||||
@@ -1774,6 +1801,11 @@ class Agent(GoalLoopMixin, SteeringMixin, SandboxMixin, SkillReviewMixin, Unifie
|
||||
toolset_tools = self._resolve_tool_names(unique_tool_names)
|
||||
self.tools.extend(toolset_tools)
|
||||
logging.debug(f"Resolved toolsets {toolsets} to {len(toolset_tools)} tools: {[getattr(t, '__name__', str(t)) for t in toolset_tools]}")
|
||||
except ToolResolutionError:
|
||||
# Preserve the typed error (with .unknown / .suggestions) so
|
||||
# strict-mode callers get the same self-correcting contract as
|
||||
# the direct tools= path instead of a flattened ValueError.
|
||||
raise
|
||||
except (ValueError, KeyError) as e:
|
||||
raise ValueError(
|
||||
f"Agent '{getattr(self, 'display_name', 'unknown')}' failed to resolve toolsets {toolsets}: {e}. "
|
||||
|
||||
@@ -727,10 +727,12 @@ Your Goal: {self.goal}"""
|
||||
Returns the possibly-mutated list of tool definitions. Synchronous
|
||||
path; use ``_aapply_before_tool_definitions_hook`` in async contexts.
|
||||
"""
|
||||
if not formatted_tools or not getattr(self, '_hook_runner', None):
|
||||
from ..hooks import HookEvent
|
||||
_runner = getattr(self, '_hook_runner', None)
|
||||
if (not formatted_tools or _runner is None
|
||||
or not _runner.registry.has_hooks(HookEvent.BEFORE_TOOL_DEFINITIONS)):
|
||||
return formatted_tools
|
||||
try:
|
||||
from ..hooks import HookEvent
|
||||
_inp = self._build_before_tool_definitions_input(formatted_tools)
|
||||
_results = self._hook_runner.execute_sync(HookEvent.BEFORE_TOOL_DEFINITIONS, _inp)
|
||||
# Fail closed on a blocking hook/plugin: a POLICY/GUARDRAIL that
|
||||
@@ -759,10 +761,12 @@ Your Goal: {self.goal}"""
|
||||
``execute_sync`` raises inside a running event loop, so the async chat
|
||||
path must await the hook runner directly to actually run the hook.
|
||||
"""
|
||||
if not formatted_tools or not getattr(self, '_hook_runner', None):
|
||||
from ..hooks import HookEvent
|
||||
_runner = getattr(self, '_hook_runner', None)
|
||||
if (not formatted_tools or _runner is None
|
||||
or not _runner.registry.has_hooks(HookEvent.BEFORE_TOOL_DEFINITIONS)):
|
||||
return formatted_tools
|
||||
try:
|
||||
from ..hooks import HookEvent
|
||||
_inp = self._build_before_tool_definitions_input(formatted_tools)
|
||||
_results = await self._hook_runner.execute(HookEvent.BEFORE_TOOL_DEFINITIONS, _inp)
|
||||
# Fail closed on a blocking hook/plugin (see sync variant).
|
||||
|
||||
@@ -20,10 +20,14 @@ from ._lazy_display import _get_console, _get_live, _get_display_functions
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
||||
from typing import List, Optional, Any, Dict, Union, Generator, TYPE_CHECKING
|
||||
|
||||
# Shared HTTP launch state (Agent.launch() multi-endpoint registration)
|
||||
_server_lock = threading.Lock()
|
||||
_server_started: Dict[int, bool] = {}
|
||||
_registered_agents: Dict[int, Dict[str, str]] = {}
|
||||
_shared_apps: Dict[int, Any] = {}
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
@@ -742,7 +746,7 @@ Write the complete compiled report:"""
|
||||
|
||||
# Show animated status during LLM call if verbose
|
||||
if self.verbose and is_tty:
|
||||
from ..main import PRAISON_COLORS, sync_display_callbacks
|
||||
from ..main import PRAISON_COLORS, sync_display_callbacks, _callbacks_lock
|
||||
import threading
|
||||
import time as time_module
|
||||
|
||||
@@ -763,9 +767,11 @@ Write the complete compiled report:"""
|
||||
tools_called.append(tool_name)
|
||||
current_status[0] = f"Calling tool: {tool_name}..."
|
||||
|
||||
# Store original callback and register ours
|
||||
original_tool_callback = sync_display_callbacks.get('tool_call')
|
||||
sync_display_callbacks['tool_call'] = status_tool_callback
|
||||
# Store original callback and register ours (use the module's
|
||||
# own lock so this doesn't race concurrent verbose agents)
|
||||
with _callbacks_lock:
|
||||
original_tool_callback = sync_display_callbacks.get('tool_call')
|
||||
sync_display_callbacks['tool_call'] = status_tool_callback
|
||||
|
||||
# Animation state
|
||||
result_holder = [None]
|
||||
@@ -803,11 +809,16 @@ Write the complete compiled report:"""
|
||||
error_holder[0] = e
|
||||
finally:
|
||||
self.verbose = original_verbose_chat
|
||||
# Restore original callback
|
||||
if original_tool_callback:
|
||||
sync_display_callbacks['tool_call'] = original_tool_callback
|
||||
elif 'tool_call' in sync_display_callbacks:
|
||||
del sync_display_callbacks['tool_call']
|
||||
# Restore original callback under the lock. The identity
|
||||
# check guards the entire restore, so we only mutate the
|
||||
# entry while it's still ours and never clobber a callback
|
||||
# a concurrent verbose agent has since installed.
|
||||
with _callbacks_lock:
|
||||
if sync_display_callbacks.get('tool_call') is status_tool_callback:
|
||||
if original_tool_callback:
|
||||
sync_display_callbacks['tool_call'] = original_tool_callback
|
||||
else:
|
||||
del sync_display_callbacks['tool_call']
|
||||
|
||||
# Start chat in background thread
|
||||
chat_thread = threading.Thread(target=run_chat)
|
||||
@@ -1143,6 +1154,68 @@ Write the complete compiled report:"""
|
||||
turn_tools = self._turn_tools_used
|
||||
turn_tools.append(function_name)
|
||||
|
||||
# Enforce BEFORE_TOOL/AFTER_TOOL security hooks for every async caller,
|
||||
# mirroring the sync execute_tool path in tool_execution.py. Without
|
||||
# this the primary async path (_execute_unified_achat_completion) would
|
||||
# silently skip HookEvent.BEFORE_TOOL gating registered via
|
||||
# Agent(hooks=HookRegistry(...)). Zero overhead when no hooks apply.
|
||||
hook_runner = getattr(self, '_hook_runner', None)
|
||||
if hook_runner is not None:
|
||||
from ..hooks import HookEvent
|
||||
if hook_runner.registry.has_hooks(HookEvent.BEFORE_TOOL):
|
||||
from ..hooks import BeforeToolInput
|
||||
before_tool_input = BeforeToolInput(
|
||||
session_id=getattr(self, '_session_id', 'default'),
|
||||
cwd=os.getcwd(),
|
||||
event_name=HookEvent.BEFORE_TOOL,
|
||||
timestamp=str(time.time()),
|
||||
agent_name=self.name,
|
||||
tool_name=function_name,
|
||||
tool_input=arguments,
|
||||
)
|
||||
before_results = await hook_runner.execute(
|
||||
HookEvent.BEFORE_TOOL, before_tool_input, target=function_name
|
||||
)
|
||||
if hook_runner.is_blocked(before_results):
|
||||
logging.warning(f"Tool {function_name} execution blocked by BEFORE_TOOL hook")
|
||||
return f"Execution of {function_name} was blocked by security policy."
|
||||
for res in before_results:
|
||||
if res.output and res.output.modified_input:
|
||||
arguments.update(res.output.modified_input)
|
||||
|
||||
result = await self._execute_tool_async_dispatch(
|
||||
function_name, arguments, tool_call_id, tools_override
|
||||
)
|
||||
|
||||
if hook_runner.registry.has_hooks(HookEvent.AFTER_TOOL):
|
||||
from ..hooks import AfterToolInput
|
||||
after_tool_input = AfterToolInput(
|
||||
session_id=getattr(self, '_session_id', 'default'),
|
||||
cwd=os.getcwd(),
|
||||
event_name=HookEvent.AFTER_TOOL,
|
||||
timestamp=str(time.time()),
|
||||
agent_name=self.name,
|
||||
tool_name=function_name,
|
||||
tool_input=arguments,
|
||||
tool_output=result,
|
||||
)
|
||||
after_results = await hook_runner.execute(
|
||||
HookEvent.AFTER_TOOL, after_tool_input, target=function_name
|
||||
)
|
||||
extra_context = hook_runner.aggregate_context(after_results)
|
||||
if extra_context:
|
||||
if isinstance(result, str):
|
||||
result = f"{result}\n\n{extra_context}"
|
||||
elif isinstance(result, dict):
|
||||
result.setdefault("_additional_context", extra_context)
|
||||
return result
|
||||
|
||||
return await self._execute_tool_async_dispatch(
|
||||
function_name, arguments, tool_call_id, tools_override
|
||||
)
|
||||
|
||||
async def _execute_tool_async_dispatch(self, function_name: str, arguments: Dict[str, Any], tool_call_id: Optional[str] = None, tools_override: Optional[List] = None) -> Any:
|
||||
"""Route an async tool call through the middleware chain (if any) then retry loop."""
|
||||
# Route async tool calls through the same tool middleware chain as the
|
||||
# sync path. The chain (before_tool/after_tool/wrap_tool_call) is
|
||||
# synchronous, so we run it in a worker thread and bridge its final
|
||||
@@ -1500,6 +1573,11 @@ Write the complete compiled report:"""
|
||||
except Exception as e:
|
||||
logging.error(f"Error starting server: {str(e)}", exc_info=True)
|
||||
print(f"❌ Error starting server: {str(e)}")
|
||||
finally:
|
||||
# Clear the started marker so a later launch() on this port
|
||||
# can retry after a bind/startup failure or server exit.
|
||||
with _server_lock:
|
||||
_server_started[port] = False
|
||||
|
||||
# Run server in a background thread
|
||||
server_thread = threading.Thread(target=run_server, daemon=True)
|
||||
|
||||
@@ -609,7 +609,9 @@ class AgentTeam(SpawnAnnounceProtocol):
|
||||
Args:
|
||||
agents: List of Agent instances
|
||||
tasks: Optional list of Task instances (auto-generated from agents if None)
|
||||
process: Execution process type ("sequential", "parallel", "hierarchical")
|
||||
process: Execution process type ("sequential", "workflow", "hierarchical").
|
||||
For parallel fan-out, set async_execution=True on individual Task
|
||||
objects within a "workflow" or "sequential" process.
|
||||
manager_llm: LLM model for manager agent
|
||||
llm: Default LLM model for all agents
|
||||
name: Name for this agent collection
|
||||
@@ -816,6 +818,14 @@ class AgentTeam(SpawnAnnounceProtocol):
|
||||
self._state_lock = threading.Lock() # Thread-safe state mutations
|
||||
self.verbose = _verbose
|
||||
self.max_retries = _max_retries
|
||||
_VALID_PROCESSES = {"workflow", "sequential", "hierarchical"}
|
||||
if process not in _VALID_PROCESSES:
|
||||
raise ValueError(
|
||||
f"Unknown process type {process!r}. Valid values are: "
|
||||
f"{sorted(_VALID_PROCESSES)}. Note: parallel fan-out is achieved "
|
||||
f"by setting async_execution=True on individual Task objects within "
|
||||
f"a 'workflow' or 'sequential' process, not via process=\"parallel\"."
|
||||
)
|
||||
self.process = process
|
||||
self.stream = _stream
|
||||
self.name = name
|
||||
@@ -1115,6 +1125,63 @@ class AgentTeam(SpawnAnnounceProtocol):
|
||||
logger.warning(f"Task {task_id}: Guardrail processing error (retry {task.retry_count}/{task.max_retries}): {e}")
|
||||
return task_output, True # Signal retry needed
|
||||
|
||||
def _run_task_start_hook(self, task, task_id):
|
||||
"""Run the on_task_start hook and propagate global variables to the task.
|
||||
|
||||
Shared by run_task (sync) and arun_task (async) so the lifecycle hooks
|
||||
and variable propagation stay consistent across both paths.
|
||||
"""
|
||||
if self.on_task_start:
|
||||
try:
|
||||
self.on_task_start(task, task_id)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in on_task_start callback: {e}")
|
||||
|
||||
# Apply global variables to task if not already set. Use a shallow copy so a
|
||||
# task mutating its variables doesn't leak back into the shared AgentTeam state.
|
||||
if self.variables and not getattr(task, 'variables', None):
|
||||
task.variables = self.variables.copy()
|
||||
|
||||
def _run_task_complete_hook(self, task, task_output):
|
||||
"""Run the on_task_complete hook. Shared by run_task and arun_task."""
|
||||
if self.on_task_complete:
|
||||
try:
|
||||
self.on_task_complete(task, task_output)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in on_task_complete callback: {e}")
|
||||
|
||||
async def _arun_task_start_hook(self, task, task_id):
|
||||
"""Async-aware on_task_start hook for arun_task.
|
||||
|
||||
Awaits coroutine callbacks and offloads synchronous ones to the executor so
|
||||
a blocking hook can't stall the event loop. Falls back to the sync helper's
|
||||
variable propagation.
|
||||
"""
|
||||
if self.on_task_start:
|
||||
try:
|
||||
if asyncio.iscoroutinefunction(self.on_task_start):
|
||||
await self.on_task_start(task, task_id)
|
||||
else:
|
||||
loop = asyncio.get_running_loop()
|
||||
await loop.run_in_executor(None, self.on_task_start, task, task_id)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in on_task_start callback: {e}")
|
||||
|
||||
if self.variables and not getattr(task, 'variables', None):
|
||||
task.variables = self.variables.copy()
|
||||
|
||||
async def _arun_task_complete_hook(self, task, task_output):
|
||||
"""Async-aware on_task_complete hook for arun_task (see _arun_task_start_hook)."""
|
||||
if self.on_task_complete:
|
||||
try:
|
||||
if asyncio.iscoroutinefunction(self.on_task_complete):
|
||||
await self.on_task_complete(task, task_output)
|
||||
else:
|
||||
loop = asyncio.get_running_loop()
|
||||
await loop.run_in_executor(None, self.on_task_complete, task, task_output)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in on_task_complete callback: {e}")
|
||||
|
||||
|
||||
async def arun_task(self, task_id):
|
||||
"""Async version of run_task method"""
|
||||
@@ -1126,6 +1193,9 @@ class AgentTeam(SpawnAnnounceProtocol):
|
||||
logger.info(f"Task with ID {task_id} is already completed")
|
||||
return
|
||||
|
||||
# Call on_task_start callback and propagate variables (async-aware, mirrors run_task)
|
||||
await self._arun_task_start_hook(task, task_id)
|
||||
|
||||
# Use per-task max_retries if available
|
||||
task_max = getattr(task, "max_retries", self.max_retries)
|
||||
retries = 0
|
||||
@@ -1154,6 +1224,10 @@ class AgentTeam(SpawnAnnounceProtocol):
|
||||
raise
|
||||
|
||||
self.save_output_to_file(task, task_output)
|
||||
|
||||
# Call on_task_complete callback (async-aware, mirrors run_task)
|
||||
await self._arun_task_complete_hook(task, task_output)
|
||||
|
||||
if self.verbose >= 1:
|
||||
logger.info(f"Task {task_id} completed successfully.")
|
||||
else:
|
||||
@@ -1349,17 +1423,9 @@ class AgentTeam(SpawnAnnounceProtocol):
|
||||
logger.info(f"Task with ID {task_id} is already completed")
|
||||
return
|
||||
|
||||
# Call on_task_start callback if provided
|
||||
if self.on_task_start:
|
||||
try:
|
||||
self.on_task_start(task, task_id)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in on_task_start callback: {e}")
|
||||
|
||||
# Apply global variables to task if not already set
|
||||
if self.variables and not getattr(task, 'variables', None):
|
||||
task.variables = self.variables
|
||||
|
||||
# Call on_task_start callback and propagate variables (shared with arun_task)
|
||||
self._run_task_start_hook(task, task_id)
|
||||
|
||||
# Use per-task max_retries if available
|
||||
task_max = getattr(task, "max_retries", self.max_retries)
|
||||
retries = 0
|
||||
@@ -1384,14 +1450,10 @@ class AgentTeam(SpawnAnnounceProtocol):
|
||||
logger.exception(e)
|
||||
|
||||
self.save_output_to_file(task, task_output)
|
||||
|
||||
# Call on_task_complete callback if provided
|
||||
if self.on_task_complete:
|
||||
try:
|
||||
self.on_task_complete(task, task_output)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in on_task_complete callback: {e}")
|
||||
|
||||
|
||||
# Call on_task_complete callback (shared with arun_task)
|
||||
self._run_task_complete_hook(task, task_output)
|
||||
|
||||
if self.verbose >= 1:
|
||||
logger.info(f"Task {task_id} completed successfully.")
|
||||
else:
|
||||
|
||||
@@ -413,6 +413,35 @@ class MessagePresentation:
|
||||
|
||||
return MessagePresentation(blocks=blocks)
|
||||
|
||||
@staticmethod
|
||||
def question(
|
||||
prompt: str,
|
||||
options: List[Any],
|
||||
context: Optional[str] = None,
|
||||
) -> "MessagePresentation":
|
||||
"""Create a structured question presentation with option buttons.
|
||||
|
||||
The symmetric counterpart to :meth:`approval` for non-binary
|
||||
clarifications ("which of these?"). Each option renders as a typed
|
||||
``reply``-action button (via :meth:`PresentationBlock.quick_replies`),
|
||||
so a tap feeds the chosen value straight back into the next agent turn
|
||||
across any channel, reusing the existing reply routing and byte-safe
|
||||
callback encoding — no new store, protocol, or correlation machinery.
|
||||
|
||||
Args:
|
||||
prompt: The question text.
|
||||
options: Choices as ``(label, value)`` pairs or plain strings.
|
||||
context: Optional context information rendered under the prompt.
|
||||
|
||||
Returns:
|
||||
A presentation with the prompt and one reply button per option.
|
||||
"""
|
||||
blocks = [PresentationBlock.make_text(prompt)]
|
||||
if context:
|
||||
blocks.append(PresentationBlock.make_context(context))
|
||||
blocks.append(PresentationBlock.quick_replies(options))
|
||||
return MessagePresentation(blocks=blocks)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PresentationLimits:
|
||||
|
||||
@@ -1438,11 +1438,22 @@ class MultiAgentMemoryConfig:
|
||||
}
|
||||
|
||||
|
||||
# Import ToolSearchConfig from tools module to avoid duplication
|
||||
def __get_tool_search_config():
|
||||
# Resolve ToolSearchConfig from tools module to avoid duplication.
|
||||
# Deferred via PEP 562 module-level __getattr__ so that `import Agent`
|
||||
# does not eagerly initialise the whole tools subsystem (see issue #3191).
|
||||
_TOOL_SEARCH_CONFIG_CACHE = None
|
||||
|
||||
|
||||
def _resolve_tool_search_config():
|
||||
# Memoize so the resolved class (including the ImportError fallback) is a
|
||||
# single stable object. Without this, the fallback path would define a new
|
||||
# dataclass per call, breaking isinstance() checks across invocations.
|
||||
global _TOOL_SEARCH_CONFIG_CACHE
|
||||
if _TOOL_SEARCH_CONFIG_CACHE is not None:
|
||||
return _TOOL_SEARCH_CONFIG_CACHE
|
||||
try:
|
||||
from ..tools.tool_search import ToolSearchConfig as _ToolSearchConfig
|
||||
return _ToolSearchConfig
|
||||
_TOOL_SEARCH_CONFIG_CACHE = _ToolSearchConfig
|
||||
except ImportError:
|
||||
# Fallback minimal config if tools module not available
|
||||
@dataclass
|
||||
@@ -1452,9 +1463,8 @@ def __get_tool_search_config():
|
||||
search_default_limit: int = 5
|
||||
max_search_limit: int = 20
|
||||
core_tools: Optional[FrozenSet[str]] = None
|
||||
return FallbackToolSearchConfig
|
||||
|
||||
ToolSearchConfig = __get_tool_search_config()
|
||||
_TOOL_SEARCH_CONFIG_CACHE = FallbackToolSearchConfig
|
||||
return _TOOL_SEARCH_CONFIG_CACHE
|
||||
|
||||
|
||||
class AutonomyLevel(str, Enum):
|
||||
@@ -1477,7 +1487,7 @@ CachingParam = Union[bool, CachingConfig]
|
||||
HooksParam = Union[List[Any], HooksConfig]
|
||||
SkillsParam = Union[List[str], SkillsConfig]
|
||||
AutonomyParam = Union[bool, Dict[str, Any], "AutonomyConfig"]
|
||||
ToolSearchParam = Union[bool, str, Dict[str, Any], ToolSearchConfig]
|
||||
ToolSearchParam = Union[bool, str, Dict[str, Any], "ToolSearchConfig"]
|
||||
ToolParam = Union[bool, ToolConfig] # bool = defaults, ToolConfig = custom
|
||||
|
||||
|
||||
@@ -1621,7 +1631,7 @@ def resolve_autonomy(value: AutonomyParam) -> Optional[AutonomyConfig]:
|
||||
return _resolve(value, AutonomyConfig)
|
||||
|
||||
|
||||
def resolve_tool_search(value: ToolSearchParam) -> Optional[ToolSearchConfig]:
|
||||
def resolve_tool_search(value: ToolSearchParam) -> Optional["ToolSearchConfig"]:
|
||||
"""
|
||||
Resolve tool_search= parameter following precedence ladder.
|
||||
|
||||
@@ -1631,6 +1641,9 @@ def resolve_tool_search(value: ToolSearchParam) -> Optional[ToolSearchConfig]:
|
||||
# Simple implementation since it's unused
|
||||
if value is None or value is False:
|
||||
return None
|
||||
# Lazy-load ToolSearchConfig only when tool search is actually configured,
|
||||
# so the tools subsystem is not imported on the `import Agent` path.
|
||||
ToolSearchConfig = _resolve_tool_search_config()
|
||||
if value is True:
|
||||
return ToolSearchConfig()
|
||||
if isinstance(value, str):
|
||||
@@ -1813,3 +1826,17 @@ __all__ = [
|
||||
"resolve_tools",
|
||||
"resolve_runtime",
|
||||
]
|
||||
|
||||
|
||||
def __getattr__(name):
|
||||
"""PEP 562 lazy attribute access.
|
||||
|
||||
``ToolSearchConfig`` is resolved (and the tools subsystem imported) only on
|
||||
first access, so ``from praisonaiagents import Agent`` does not eagerly load
|
||||
the entire tools package (issue #3191).
|
||||
"""
|
||||
if name == "ToolSearchConfig":
|
||||
cfg = _resolve_tool_search_config()
|
||||
globals()["ToolSearchConfig"] = cfg # cache for subsequent access
|
||||
return cfg
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -72,8 +72,29 @@ class ContextAggregator:
|
||||
self._sources: Dict[str, Tuple[Callable, int]] = {}
|
||||
|
||||
def _estimate_tokens(self, text: str) -> int:
|
||||
"""Estimate token count (4 chars per token heuristic)."""
|
||||
return len(text) // 4 + 1
|
||||
"""Estimate token count via the canonical heuristic."""
|
||||
from .tokens import estimate_tokens_heuristic
|
||||
return estimate_tokens_heuristic(text)
|
||||
|
||||
def _truncate_to_tokens(self, text: str, max_tokens: int) -> str:
|
||||
"""Return the longest prefix of ``text`` whose estimated tokens fit ``max_tokens``.
|
||||
|
||||
Uses the canonical estimator so dense non-ASCII scripts (e.g. CJK) are
|
||||
truncated correctly rather than assuming a flat 4-chars-per-token ratio.
|
||||
"""
|
||||
if max_tokens <= 0 or not text:
|
||||
return ""
|
||||
if self._estimate_tokens(text) <= max_tokens:
|
||||
return text
|
||||
# Binary search for the longest prefix within budget.
|
||||
lo, hi = 0, len(text)
|
||||
while lo < hi:
|
||||
mid = (lo + hi + 1) // 2
|
||||
if self._estimate_tokens(text[:mid]) <= max_tokens:
|
||||
lo = mid
|
||||
else:
|
||||
hi = mid - 1
|
||||
return text[:lo]
|
||||
|
||||
def register_source(
|
||||
self,
|
||||
@@ -219,9 +240,15 @@ class ContextAggregator:
|
||||
# Try to fit partial
|
||||
remaining = max_tokens - total_tokens - separator_tokens
|
||||
if remaining > 100:
|
||||
# Truncate context
|
||||
chars_to_keep = int(remaining * 4 * 0.9)
|
||||
truncated = context[:chars_to_keep] + "..."
|
||||
# Reserve budget for the label and ellipsis so the retained
|
||||
# prefix (canonically estimated) stays within ``remaining``.
|
||||
ellipsis_tokens = self._estimate_tokens("...")
|
||||
label_tokens = (
|
||||
self._estimate_tokens(f"**{name}**:\n")
|
||||
if self.include_source_labels else 0
|
||||
)
|
||||
prefix_budget = remaining - ellipsis_tokens - label_tokens
|
||||
truncated = self._truncate_to_tokens(context, prefix_budget) + "..."
|
||||
|
||||
if self.include_source_labels:
|
||||
merged_parts.append(f"**{name}**:\n{truncated}")
|
||||
|
||||
@@ -22,26 +22,22 @@ import logging
|
||||
from praisonaiagents._logging import get_logger
|
||||
from typing import Dict, Optional, Union
|
||||
|
||||
from ..context.budgeter import MODEL_LIMITS as _CANONICAL_LIMITS
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Default context lengths for common models (fallback when litellm unavailable)
|
||||
# Based on official documentation as of January 2025
|
||||
DEFAULT_CONTEXT_LENGTHS: Dict[str, int] = {
|
||||
# Eval-specific context lengths not present in the canonical budgeter table
|
||||
# (dated Anthropic snapshots, o1 variants, Mistral/DeepSeek/Groq, etc.).
|
||||
# Based on official documentation as of January 2025.
|
||||
_EVAL_EXTRA_LENGTHS: Dict[str, int] = {
|
||||
# OpenAI models
|
||||
"gpt-4o": 128000,
|
||||
"gpt-4o-mini": 128000,
|
||||
"gpt-4-turbo": 128000,
|
||||
"gpt-4-turbo-preview": 128000,
|
||||
"gpt-4": 8192,
|
||||
"gpt-4-32k": 32768,
|
||||
"gpt-3.5-turbo": 16385,
|
||||
"gpt-3.5-turbo-16k": 16385,
|
||||
"o1": 200000,
|
||||
"o1-mini": 128000,
|
||||
"o1-preview": 128000,
|
||||
"o3-mini": 200000,
|
||||
|
||||
# Anthropic models
|
||||
|
||||
# Anthropic models (dated snapshots)
|
||||
"claude-3-5-sonnet-20241022": 200000,
|
||||
"claude-3-5-sonnet-latest": 200000,
|
||||
"claude-3-5-haiku-20241022": 200000,
|
||||
@@ -50,24 +46,22 @@ DEFAULT_CONTEXT_LENGTHS: Dict[str, int] = {
|
||||
"claude-3-haiku-20240307": 200000,
|
||||
"claude-2.1": 200000,
|
||||
"claude-2": 100000,
|
||||
|
||||
|
||||
# Google models
|
||||
"gemini-1.5-pro": 2097152,
|
||||
"gemini-1.5-flash": 1048576,
|
||||
"gemini-1.5-flash-8b": 1048576,
|
||||
"gemini-2.0-flash-exp": 1048576,
|
||||
"gemini-pro": 32760,
|
||||
|
||||
|
||||
# Mistral models
|
||||
"mistral-large-latest": 128000,
|
||||
"mistral-medium-latest": 32000,
|
||||
"mistral-small-latest": 32000,
|
||||
"codestral-latest": 32000,
|
||||
|
||||
|
||||
# DeepSeek models
|
||||
"deepseek-chat": 64000,
|
||||
"deepseek-coder": 64000,
|
||||
|
||||
|
||||
# Groq models
|
||||
"llama-3.3-70b-versatile": 128000,
|
||||
"llama-3.1-70b-versatile": 128000,
|
||||
@@ -75,6 +69,36 @@ DEFAULT_CONTEXT_LENGTHS: Dict[str, int] = {
|
||||
"mixtral-8x7b-32768": 32768,
|
||||
}
|
||||
|
||||
# Guard the invariant that eval-only extras never silently shadow canonical
|
||||
# entries. If a key is later added to the canonical budgeter table, it should be
|
||||
# removed from _EVAL_EXTRA_LENGTHS so the single source of truth stays authoritative.
|
||||
_overlapping_keys = set(_EVAL_EXTRA_LENGTHS) & {
|
||||
k for k in _CANONICAL_LIMITS if k != "default"
|
||||
}
|
||||
assert not _overlapping_keys, (
|
||||
"_EVAL_EXTRA_LENGTHS keys overlap with canonical MODEL_LIMITS; remove the "
|
||||
f"duplicate(s) from the eval table: {sorted(_overlapping_keys)}"
|
||||
)
|
||||
|
||||
# Default context lengths for common models (fallback when litellm unavailable).
|
||||
# Built by merging the canonical budgeter table (context/budgeter.MODEL_LIMITS)
|
||||
# with eval-specific extras, so a single source of truth is maintained and new
|
||||
# model families (e.g. gpt-5, gpt-4.1) resolve consistently across both paths.
|
||||
#
|
||||
# Keys are ordered by descending length so that get_context_length's partial
|
||||
# matching checks specific names (e.g. "gpt-4-32k") before shorter prefixes
|
||||
# (e.g. "gpt-4") and returns the most precise context window.
|
||||
DEFAULT_CONTEXT_LENGTHS: Dict[str, int] = dict(
|
||||
sorted(
|
||||
{
|
||||
**{k: v for k, v in _CANONICAL_LIMITS.items() if k != "default"},
|
||||
**_EVAL_EXTRA_LENGTHS,
|
||||
}.items(),
|
||||
key=lambda item: len(item[0]),
|
||||
reverse=True,
|
||||
)
|
||||
)
|
||||
|
||||
# Default context length for unknown models
|
||||
DEFAULT_CONTEXT_LENGTH = 128000
|
||||
|
||||
|
||||
@@ -21,6 +21,11 @@ from .protocols import (
|
||||
EventType,
|
||||
OperatorScope,
|
||||
GatewayCloseCode,
|
||||
# Declarative method -> required-scope registry (Issue #3206)
|
||||
GatewayMethodDescriptor,
|
||||
GATEWAY_METHODS,
|
||||
register_gateway_method,
|
||||
resolve_required_scope,
|
||||
# Config hot-reload observability (Issue #3049)
|
||||
ReloadStatus,
|
||||
compute_config_revision,
|
||||
@@ -126,6 +131,11 @@ from .protocols import (
|
||||
FrameDecodeError,
|
||||
ClientFrame,
|
||||
decode_client_frame,
|
||||
# Weak / placeholder secret guard (Issue #3259)
|
||||
KNOWN_WEAK_SECRETS,
|
||||
WeakGatewaySecretError,
|
||||
is_weak_secret,
|
||||
assert_gateway_secret_strong,
|
||||
)
|
||||
from .hooks import (
|
||||
HookAction,
|
||||
@@ -133,6 +143,7 @@ from .hooks import (
|
||||
InboundTriggerProtocol,
|
||||
render_template,
|
||||
compute_idempotency_key,
|
||||
verify_webhook_signature,
|
||||
)
|
||||
from .config import (
|
||||
GatewayConfig,
|
||||
@@ -209,6 +220,11 @@ __all__ = [
|
||||
"EventType",
|
||||
"OperatorScope",
|
||||
"GatewayCloseCode",
|
||||
# Declarative method -> required-scope registry (Issue #3206)
|
||||
"GatewayMethodDescriptor",
|
||||
"GATEWAY_METHODS",
|
||||
"register_gateway_method",
|
||||
"resolve_required_scope",
|
||||
# Config hot-reload observability (Issue #3049)
|
||||
"ReloadStatus",
|
||||
"compute_config_revision",
|
||||
@@ -313,12 +329,18 @@ __all__ = [
|
||||
"FrameDecodeError",
|
||||
"ClientFrame",
|
||||
"decode_client_frame",
|
||||
# Weak / placeholder secret guard (Issue #3259)
|
||||
"KNOWN_WEAK_SECRETS",
|
||||
"WeakGatewaySecretError",
|
||||
"is_weak_secret",
|
||||
"assert_gateway_secret_strong",
|
||||
# Inbound trigger / webhook contract (Issue #2281)
|
||||
"HookAction",
|
||||
"HookConfig",
|
||||
"InboundTriggerProtocol",
|
||||
"render_template",
|
||||
"compute_idempotency_key",
|
||||
"verify_webhook_signature",
|
||||
# Config (always available)
|
||||
"GatewayConfig",
|
||||
"SessionConfig",
|
||||
|
||||
@@ -27,9 +27,10 @@ process; this fires from an external HTTP request.
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, Optional, Protocol, runtime_checkable
|
||||
from typing import Any, Dict, List, Optional, Protocol, runtime_checkable
|
||||
|
||||
__all__ = [
|
||||
"HookAction",
|
||||
@@ -37,6 +38,7 @@ __all__ = [
|
||||
"InboundTriggerProtocol",
|
||||
"compute_idempotency_key",
|
||||
"render_template",
|
||||
"verify_webhook_signature",
|
||||
]
|
||||
|
||||
|
||||
@@ -148,6 +150,56 @@ def compute_idempotency_key(
|
||||
return hashlib.sha256(basis.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def verify_webhook_signature(
|
||||
secret: Optional[str],
|
||||
raw_body: bytes,
|
||||
signature: Optional[str],
|
||||
*,
|
||||
algo: str = "sha256",
|
||||
prefix: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""Constant-time HMAC verification of a webhook body (dependency-free).
|
||||
|
||||
A pure counterpart to ``compute_idempotency_key`` that lets the core
|
||||
contract validate a provider signature over the *raw* request bytes without
|
||||
importing the wrapper's crypto. Mirrors the wrapper's ``verify_hmac``: it is
|
||||
fail-closed (a missing secret/signature or unknown ``algo`` returns
|
||||
``False`` rather than raising) and prefix-aware.
|
||||
|
||||
Args:
|
||||
secret: Shared signing secret. Falsy → ``False``.
|
||||
raw_body: The exact raw request body bytes the provider signed.
|
||||
signature: The signature header value the provider sent.
|
||||
algo: Hash algorithm name (e.g. ``"sha256"``, ``"sha1"``).
|
||||
prefix: Optional signature prefix (e.g. ``"sha256="``). When provided,
|
||||
the comparison is against the fully-prefixed computed value, so the
|
||||
caller passes the raw header value unchanged. When omitted, an
|
||||
``algo=`` style prefix on the provided signature is auto-stripped.
|
||||
|
||||
Returns:
|
||||
``True`` only if a non-empty signature matches the computed HMAC using a
|
||||
constant-time comparison.
|
||||
"""
|
||||
if not secret or not signature:
|
||||
return False
|
||||
|
||||
secret_bytes = secret.encode("utf-8") if isinstance(secret, str) else secret
|
||||
body_bytes = raw_body if isinstance(raw_body, bytes) else str(raw_body).encode("utf-8")
|
||||
|
||||
try:
|
||||
computed = hmac.new(secret_bytes, body_bytes, algo).hexdigest()
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
provided = signature
|
||||
if prefix:
|
||||
return hmac.compare_digest(f"{prefix}{computed}", provided)
|
||||
|
||||
if "=" in provided and provided.split("=", 1)[0].isalnum():
|
||||
provided = provided.split("=", 1)[1]
|
||||
return hmac.compare_digest(computed, provided)
|
||||
|
||||
|
||||
@dataclass
|
||||
class HookConfig:
|
||||
"""Declarative definition of an inbound event trigger.
|
||||
@@ -172,6 +224,20 @@ class HookConfig:
|
||||
message: Template for the message built from the payload.
|
||||
enabled: Whether the hook is active.
|
||||
metadata: Free-form extra settings.
|
||||
secret: Optional HMAC signing secret. When set, the gateway verifies the
|
||||
provider signature over the *raw* body before any agent runs and
|
||||
rejects (401) a missing/invalid signature — fail-closed.
|
||||
signature_header: Header carrying the signature, e.g.
|
||||
``"X-Hub-Signature-256"``.
|
||||
signature_algo: Digest for the HMAC, e.g. ``"sha256"``.
|
||||
signature_prefix: Optional signature prefix, e.g. ``"sha256="``.
|
||||
events: Optional allow-list of event types. A delivery whose event is
|
||||
not listed is acknowledged (200) without running a turn.
|
||||
event_header: Header carrying the event type, e.g. ``"X-GitHub-Event"``.
|
||||
When omitted the event is read from the payload (dotted path) via
|
||||
``resolve_event``.
|
||||
deliver_only: When ``True`` the rendered ``message`` *is* the delivered
|
||||
content, routed straight through ``deliver_to`` with no LLM turn.
|
||||
"""
|
||||
|
||||
path: str
|
||||
@@ -184,6 +250,16 @@ class HookConfig:
|
||||
message: Optional[str] = None
|
||||
enabled: bool = True
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
# Signature verification (over the RAW body, before the agent runs).
|
||||
secret: Optional[str] = None
|
||||
signature_header: Optional[str] = None
|
||||
signature_algo: str = "sha256"
|
||||
signature_prefix: Optional[str] = None
|
||||
# Event filtering.
|
||||
events: Optional[List[str]] = None
|
||||
event_header: Optional[str] = None
|
||||
# Pass-through: deliver the rendered message with no LLM turn.
|
||||
deliver_only: bool = False
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.path = (self.path or "").strip().strip("/")
|
||||
@@ -194,6 +270,87 @@ class HookConfig:
|
||||
f"HookConfig.action must be one of {HookAction.all()}, "
|
||||
f"got {self.action!r}"
|
||||
)
|
||||
if isinstance(self.events, str):
|
||||
self.events = [self.events]
|
||||
# A configured secret with no explicit header would otherwise read no
|
||||
# signature and reject every request. Default to the widely-used
|
||||
# ``X-Hub-Signature-256`` (GitHub/webhook convention) so ``secret`` on
|
||||
# its own is a working, verifying configuration rather than a 401 trap.
|
||||
if self.secret and not self.signature_header:
|
||||
self.signature_header = "X-Hub-Signature-256"
|
||||
|
||||
def verify_signature(
|
||||
self, raw_body: bytes, headers: Dict[str, str]
|
||||
) -> bool:
|
||||
"""Verify the provider HMAC signature over ``raw_body``.
|
||||
|
||||
Returns ``True`` when no ``secret`` is configured (signature checking is
|
||||
opt-in); otherwise delegates to :func:`verify_webhook_signature`, which
|
||||
is fail-closed on a missing/invalid signature.
|
||||
"""
|
||||
if not self.secret:
|
||||
return True
|
||||
signature = None
|
||||
if self.signature_header:
|
||||
lowered = {k.lower(): v for k, v in headers.items()}
|
||||
signature = lowered.get(self.signature_header.lower())
|
||||
return verify_webhook_signature(
|
||||
self.secret,
|
||||
raw_body,
|
||||
signature,
|
||||
algo=self.signature_algo,
|
||||
prefix=self.signature_prefix,
|
||||
)
|
||||
|
||||
def resolve_event(
|
||||
self, payload: Dict[str, Any], headers: Optional[Dict[str, str]] = None
|
||||
) -> str:
|
||||
"""Resolve the event type from a header or the payload.
|
||||
|
||||
Reads ``event_header`` from ``headers`` when configured, else treats
|
||||
``event_header`` as a dotted payload path (defaulting to ``"event"``).
|
||||
Returns the base event name only (e.g. GitHub's ``"issues"``); a
|
||||
payload ``action`` sub-type (``"issues.opened"``) is matched separately
|
||||
by :meth:`event_allowed` so both ``issues`` and ``issues.opened`` work.
|
||||
"""
|
||||
if self.event_header and headers:
|
||||
lowered = {k.lower(): v for k, v in headers.items()}
|
||||
value = lowered.get(self.event_header.lower())
|
||||
if value:
|
||||
return str(value)
|
||||
path = self.event_header or "event"
|
||||
value = _lookup(path, payload)
|
||||
return "" if value is None else str(value)
|
||||
|
||||
def event_allowed(
|
||||
self, payload: Dict[str, Any], headers: Optional[Dict[str, str]] = None
|
||||
) -> bool:
|
||||
"""Whether the delivery's event passes the configured ``events`` filter.
|
||||
|
||||
Returns ``True`` when no filter is set. The resolved base event (e.g.
|
||||
GitHub's ``"issues"``) matches a listed name that is equal to it or is
|
||||
namespaced under it (``"issues.opened"``); in the namespaced case the
|
||||
payload's ``action`` (when present) must equal the sub-type, so
|
||||
``events: [issues.opened]`` accepts an ``issues`` delivery only when
|
||||
``action == "opened"``.
|
||||
"""
|
||||
if not self.events:
|
||||
return True
|
||||
event = self.resolve_event(payload, headers)
|
||||
if not event:
|
||||
return False
|
||||
action = payload.get("action") if isinstance(payload, dict) else None
|
||||
for allowed in self.events:
|
||||
if event == allowed:
|
||||
return True
|
||||
base, sep, sub = allowed.partition(".")
|
||||
# Namespaced sub-type (e.g. ``issues.opened``): fail-closed — the
|
||||
# payload must actually carry the matching ``action``. A delivery
|
||||
# that omits ``action`` is NOT admitted, so an ``issues`` event
|
||||
# cannot slip through a filter that only allows ``issues.opened``.
|
||||
if sep and base == event and action is not None and str(action) == sub:
|
||||
return True
|
||||
return False
|
||||
|
||||
@property
|
||||
def route(self) -> str:
|
||||
@@ -223,7 +380,7 @@ class HookConfig:
|
||||
return render_template(self.message, payload)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to a dictionary (hides the auth secret)."""
|
||||
"""Convert to a dictionary (hides the auth/signing secrets)."""
|
||||
return {
|
||||
"path": self.path,
|
||||
"agent": self.agent,
|
||||
@@ -235,6 +392,13 @@ class HookConfig:
|
||||
"message": self.message,
|
||||
"enabled": self.enabled,
|
||||
"metadata": dict(self.metadata),
|
||||
"secret": "***" if self.secret else None,
|
||||
"signature_header": self.signature_header,
|
||||
"signature_algo": self.signature_algo,
|
||||
"signature_prefix": self.signature_prefix,
|
||||
"events": list(self.events) if self.events else None,
|
||||
"event_header": self.event_header,
|
||||
"deliver_only": self.deliver_only,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
@@ -251,6 +415,13 @@ class HookConfig:
|
||||
"message",
|
||||
"enabled",
|
||||
"metadata",
|
||||
"secret",
|
||||
"signature_header",
|
||||
"signature_algo",
|
||||
"signature_prefix",
|
||||
"events",
|
||||
"event_header",
|
||||
"deliver_only",
|
||||
}
|
||||
return cls(
|
||||
path=data.get("path", ""),
|
||||
@@ -262,6 +433,13 @@ class HookConfig:
|
||||
deliver_to=data.get("deliver_to"),
|
||||
message=data.get("message"),
|
||||
enabled=data.get("enabled", True),
|
||||
secret=data.get("secret"),
|
||||
signature_header=data.get("signature_header"),
|
||||
signature_algo=data.get("signature_algo", "sha256"),
|
||||
signature_prefix=data.get("signature_prefix"),
|
||||
events=data.get("events"),
|
||||
event_header=data.get("event_header"),
|
||||
deliver_only=data.get("deliver_only", False),
|
||||
metadata={
|
||||
**(data.get("metadata") or {}),
|
||||
**{k: v for k, v in data.items() if k not in known},
|
||||
|
||||
@@ -1685,6 +1685,12 @@ class RouteBinding:
|
||||
``standard`` / ``trusted`` apply no tier deny-list.
|
||||
allow_tools: If set, only these tool names are exposed on this route.
|
||||
deny_tools: Tool names removed before the run on this route.
|
||||
profile: Optional isolated tenant-profile name (Issue #3189). Names a
|
||||
per-route isolation scope the wrapper enters for the turn (e.g. its
|
||||
own memory namespace / secret scope / home), so one gateway can
|
||||
safely multiplex tenants. ``None`` means the route is unscoped; the
|
||||
wrapper must fail closed (never fall back to another tenant's
|
||||
profile) rather than silently share memory/secrets.
|
||||
"""
|
||||
|
||||
agent: str
|
||||
@@ -1697,6 +1703,7 @@ class RouteBinding:
|
||||
trust: Optional[str] = None
|
||||
allow_tools: Optional[List[str]] = None
|
||||
deny_tools: Optional[List[str]] = None
|
||||
profile: Optional[str] = None
|
||||
|
||||
# Specificity weights — exact peer beats role/channel beats account
|
||||
# beats chat-type. Higher means more specific.
|
||||
@@ -1709,13 +1716,21 @@ class RouteBinding:
|
||||
}
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Normalise ``trust`` so config typos cannot silently fail open.
|
||||
"""Normalise ``trust``/``profile`` so config typos cannot fail open.
|
||||
|
||||
Whitespace/case variants of a known tier (e.g. ``" Untrusted "``) are
|
||||
canonicalised. Any *unknown* non-empty value is treated as the most
|
||||
restrictive tier (``untrusted``) rather than as "no policy", so a
|
||||
misconfigured route can never accidentally expose the full toolset.
|
||||
A blank ``profile`` is coerced to ``None`` (unscoped) for the same
|
||||
fail-closed reason.
|
||||
"""
|
||||
# Blank/whitespace-only profile means "unscoped" (None), never an
|
||||
# empty-named scope, so a wrapper checking ``if profile is not None``
|
||||
# fails closed rather than entering an anonymous namespace.
|
||||
if self.profile is not None and not str(self.profile).strip():
|
||||
self.profile = None
|
||||
|
||||
if self.trust is None:
|
||||
return
|
||||
normalized = str(self.trust).strip().lower()
|
||||
@@ -1792,6 +1807,7 @@ class RouteBinding:
|
||||
trust=_as_opt_str(data.get("trust")),
|
||||
allow_tools=_as_opt_str_list(data.get("allow_tools")),
|
||||
deny_tools=_as_opt_str_list(data.get("deny_tools")),
|
||||
profile=_as_opt_str(data.get("profile")),
|
||||
)
|
||||
|
||||
|
||||
@@ -1822,11 +1838,18 @@ class RouteMatch:
|
||||
agent: The resolved agent id.
|
||||
binding: The binding that matched, or ``None`` when the fallback was used.
|
||||
reason: Short human-readable explanation for logging/debugging.
|
||||
profile: The isolated tenant-profile named by the matched binding, or
|
||||
``None`` when the route is unscoped / the fallback was used (Issue
|
||||
#3189). Surfaced here so the wrapper can enter the profile's memory
|
||||
namespace / secret scope for the turn without re-resolving, and so
|
||||
an unmatched route fails closed (never inherits another tenant's
|
||||
profile).
|
||||
"""
|
||||
|
||||
agent: str
|
||||
binding: Optional[RouteBinding] = None
|
||||
reason: str = ""
|
||||
profile: Optional[str] = None
|
||||
|
||||
|
||||
def _as_opt_str(value: Any) -> Optional[str]:
|
||||
@@ -1892,6 +1915,7 @@ def resolve_route(
|
||||
f"matched binding (priority={best.priority}, "
|
||||
f"specificity={best.specificity})"
|
||||
),
|
||||
profile=best.profile,
|
||||
)
|
||||
|
||||
return RouteMatch(
|
||||
@@ -1974,6 +1998,71 @@ def resolve_auth_mode(bind_host: str, configured: Optional[AuthMode] = None) ->
|
||||
return "local" if is_loopback(bind_host) else "token"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Weak / placeholder secret guard (Issue #3259)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
KNOWN_WEAK_SECRETS: frozenset = frozenset({
|
||||
"change-me", "changeme", "change-me-now", "changemenow",
|
||||
"your-token-here", "your_token_here", "your-secret-here",
|
||||
"secret", "password", "passwd", "test", "token", "admin",
|
||||
"default", "example", "placeholder", "none", "null", "todo",
|
||||
# Copy-paste footgun: the literal fix-hint command pasted verbatim.
|
||||
"$(openssl rand -hex 16)", "$(openssl rand -hex 32)",
|
||||
})
|
||||
"""Well-known placeholder/weak shared secrets that must never protect a gateway.
|
||||
|
||||
A gateway bound to an external interface and "protected" by one of these
|
||||
publicly-known values is effectively unauthenticated. See Issue #3259.
|
||||
"""
|
||||
|
||||
|
||||
class WeakGatewaySecretError(Exception):
|
||||
"""Raised when a gateway secret matches a known-weak/placeholder value."""
|
||||
|
||||
def __init__(self, field: str = "gateway.auth_token"):
|
||||
self.field = field
|
||||
super().__init__(
|
||||
f"Refusing to start: {field} is a known-weak/placeholder value.\n"
|
||||
f"A publicly-known secret provides no real authentication.\n"
|
||||
f"Fix: praisonai onboard (30 seconds, 3 prompts)\n"
|
||||
f"Or: export GATEWAY_AUTH_TOKEN=\"$(openssl rand -hex 16)\" "
|
||||
f"(run in a shell so the command is expanded, not pasted literally)"
|
||||
)
|
||||
|
||||
|
||||
def is_weak_secret(value: Optional[str]) -> bool:
|
||||
"""Return True if ``value`` is empty or a known-weak/placeholder secret.
|
||||
|
||||
Comparison is whitespace-stripped and case-insensitive.
|
||||
|
||||
Examples:
|
||||
>>> is_weak_secret("change-me")
|
||||
True
|
||||
>>> is_weak_secret("$(openssl rand -hex 16)")
|
||||
True
|
||||
>>> is_weak_secret("strong-non-placeholder-token")
|
||||
False
|
||||
"""
|
||||
if not value:
|
||||
return True
|
||||
return str(value).strip().lower() in KNOWN_WEAK_SECRETS
|
||||
|
||||
|
||||
def assert_gateway_secret_strong(value: Optional[str], *, field: str = "gateway.auth_token") -> None:
|
||||
"""Fail closed if ``value`` is a known-weak/placeholder gateway secret.
|
||||
|
||||
Args:
|
||||
value: The resolved secret to validate.
|
||||
field: Name of the credential (used in the error message).
|
||||
|
||||
Raises:
|
||||
WeakGatewaySecretError: If ``value`` matches a known-weak value.
|
||||
"""
|
||||
if is_weak_secret(value):
|
||||
raise WeakGatewaySecretError(field=field)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth, Pairing, and Session Binding Protocols (Issue #1588 Gap 3)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -4303,3 +4392,216 @@ class LivenessPolicy:
|
||||
if now > self.reap_deadline(last_activity):
|
||||
return LivenessDecision.REAP
|
||||
return LivenessDecision.KEEP
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Declarative method -> required-scope registry (Issue #3206)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GatewayMethodDescriptor:
|
||||
"""Declarative authorisation descriptor for one gateway method.
|
||||
|
||||
A descriptor states — once, next to the protocol it guards — *what scope a
|
||||
caller must hold* to invoke ``name``. The dispatcher resolves the required
|
||||
scope from the registry instead of scattering ``_require_scope`` /
|
||||
``_client_has_scope`` calls per endpoint, so a newly added method is
|
||||
**closed until explicitly classified** rather than reachable by omission.
|
||||
|
||||
Attributes:
|
||||
name: The method / route / message-type identifier (e.g.
|
||||
``"agent.message"``, ``"channels.control"``).
|
||||
required_scope: The baseline scope required to invoke the method.
|
||||
owner: Who declared the method (``"core"``, a plugin name, ...). Purely
|
||||
informational; helps auditing a growing control surface.
|
||||
since: Optional version/date the method was classified.
|
||||
escalate_fields: Optional per-payload-field escalation. Maps a param
|
||||
field name to the stricter scope demanded when that field is
|
||||
present. Fields not listed here are treated as *unknown/structural*
|
||||
and escalate to :attr:`escalate_unknown_scope` (fail closed) when
|
||||
:attr:`strict_fields` is True.
|
||||
strict_fields: When True, any payload field not in ``escalate_fields``
|
||||
(and not in ``safe_fields``) escalates to
|
||||
:attr:`escalate_unknown_scope`. Defaults to False so today's
|
||||
behaviour (no field-level derivation) is preserved unless opted in.
|
||||
safe_fields: Field names that never escalate — read-only / benign
|
||||
params. Only consulted when ``strict_fields`` is True.
|
||||
escalate_unknown_scope: Scope demanded for unknown structural fields
|
||||
under ``strict_fields``. Defaults to ``ADMIN`` (fail closed).
|
||||
"""
|
||||
|
||||
name: str
|
||||
required_scope: OperatorScope = OperatorScope.ADMIN
|
||||
owner: str = "core"
|
||||
since: Optional[str] = None
|
||||
escalate_fields: Dict[str, OperatorScope] = field(default_factory=dict)
|
||||
strict_fields: bool = False
|
||||
safe_fields: Set[str] = field(default_factory=set)
|
||||
escalate_unknown_scope: OperatorScope = OperatorScope.ADMIN
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Defensively copy the mutable collections so the descriptor is a
|
||||
genuinely immutable authorisation record.
|
||||
|
||||
``@dataclass(frozen=True)`` blocks attribute *reassignment* but not
|
||||
in-place mutation of the referenced dict/set. Direct construction
|
||||
(e.g. a plugin building a descriptor without going through
|
||||
:func:`register_gateway_method`) could otherwise mutate
|
||||
``escalate_fields`` / ``safe_fields`` after the fact and silently
|
||||
change the resolved scope. Freeze them at construction so the resolved
|
||||
scope can never drift after registration.
|
||||
"""
|
||||
object.__setattr__(self, "escalate_fields", dict(self.escalate_fields))
|
||||
object.__setattr__(self, "safe_fields", frozenset(self.safe_fields))
|
||||
|
||||
def resolve(self, params: Optional[Dict[str, Any]] = None) -> OperatorScope:
|
||||
"""Resolve the effective required scope for a call with ``params``.
|
||||
|
||||
Starts from :attr:`required_scope` and escalates (never de-escalates)
|
||||
based on the payload:
|
||||
|
||||
* any field listed in :attr:`escalate_fields` raises the requirement
|
||||
to that field's scope;
|
||||
* under :attr:`strict_fields`, any field that is neither a
|
||||
``safe_field`` nor an ``escalate_field`` is treated as unknown /
|
||||
structural and raises the requirement to
|
||||
:attr:`escalate_unknown_scope` (fail closed on unknown fields).
|
||||
"""
|
||||
required = self.required_scope
|
||||
if params:
|
||||
for key in params:
|
||||
escalated = self.escalate_fields.get(key)
|
||||
if escalated is not None:
|
||||
required = _max_scope(required, escalated)
|
||||
elif self.strict_fields and key not in self.safe_fields:
|
||||
required = _max_scope(required, self.escalate_unknown_scope)
|
||||
return required
|
||||
|
||||
|
||||
# Privilege lattice used to combine (never weaken) scopes.
|
||||
#
|
||||
# READ < WRITE are a strict linear chain. APPROVALS and PAIRING are *sibling*
|
||||
# capabilities at the same tier: each is stricter than WRITE, but they are
|
||||
# **incomparable** to each other (holding APPROVALS does not imply PAIRING or
|
||||
# vice versa). ADMIN is the top of the lattice.
|
||||
#
|
||||
# ``_SCOPE_TIER`` gives the linear rank; APPROVALS and PAIRING share a tier only
|
||||
# to express "both above WRITE, both below ADMIN". Combining two *distinct*
|
||||
# scopes that sit at that same tier is unsound to collapse into one of them, so
|
||||
# :func:`_max_scope` escalates such a pair to their common upper bound, ADMIN
|
||||
# (fail closed) rather than silently picking whichever was passed first.
|
||||
_SCOPE_TIER: Dict[OperatorScope, int] = {
|
||||
OperatorScope.READ: 0,
|
||||
OperatorScope.WRITE: 1,
|
||||
OperatorScope.APPROVALS: 2,
|
||||
OperatorScope.PAIRING: 2,
|
||||
OperatorScope.ADMIN: 3,
|
||||
}
|
||||
|
||||
# Scopes at a shared tier that are siblings (incomparable), so combining two
|
||||
# different ones must escalate rather than pick one.
|
||||
_INCOMPARABLE_TIERS = frozenset({2})
|
||||
|
||||
|
||||
def _max_scope(a: OperatorScope, b: OperatorScope) -> OperatorScope:
|
||||
"""Return the stricter scope, escalating incomparable siblings to ADMIN.
|
||||
|
||||
For the linear part of the lattice (READ < WRITE < ... < ADMIN) this
|
||||
returns the higher-ranked scope. When ``a`` and ``b`` are two *distinct*
|
||||
scopes sharing an incomparable tier (e.g. APPROVALS vs PAIRING), neither
|
||||
implies the other, so the combined requirement is escalated to their common
|
||||
upper bound (``ADMIN``) to stay fail-closed — a single-scope check can then
|
||||
never be satisfied by holding only one of the two required capabilities.
|
||||
"""
|
||||
if a == b:
|
||||
return a
|
||||
tier_a = _SCOPE_TIER.get(a, _SCOPE_TIER[OperatorScope.ADMIN])
|
||||
tier_b = _SCOPE_TIER.get(b, _SCOPE_TIER[OperatorScope.ADMIN])
|
||||
if tier_a == tier_b and tier_a in _INCOMPARABLE_TIERS:
|
||||
return OperatorScope.ADMIN
|
||||
return b if tier_b > tier_a else a
|
||||
|
||||
|
||||
# Module-level registry. Kept intentionally simple (a dict) so the contract is
|
||||
# a pure, dependency-free lookup that clients and the wrapper can share.
|
||||
GATEWAY_METHODS: Dict[str, GatewayMethodDescriptor] = {}
|
||||
|
||||
|
||||
def register_gateway_method(
|
||||
name: str,
|
||||
*,
|
||||
scope: OperatorScope = OperatorScope.ADMIN,
|
||||
owner: str = "core",
|
||||
since: Optional[str] = None,
|
||||
escalate_fields: Optional[Dict[str, OperatorScope]] = None,
|
||||
strict_fields: bool = False,
|
||||
safe_fields: Optional[Set[str]] = None,
|
||||
escalate_unknown_scope: OperatorScope = OperatorScope.ADMIN,
|
||||
replace: bool = False,
|
||||
) -> GatewayMethodDescriptor:
|
||||
"""Register (once) the required scope for a gateway method.
|
||||
|
||||
Core methods are registered at import time (see below); plugins that add
|
||||
new gateway surface should register their descriptors through this same
|
||||
function so they inherit default-deny semantics.
|
||||
|
||||
Raises:
|
||||
ValueError: If ``name`` is already registered and ``replace`` is False.
|
||||
"""
|
||||
if not replace and name in GATEWAY_METHODS:
|
||||
raise ValueError(
|
||||
f"gateway method {name!r} is already registered "
|
||||
f"(pass replace=True to override)"
|
||||
)
|
||||
desc = GatewayMethodDescriptor(
|
||||
name=name,
|
||||
required_scope=scope,
|
||||
owner=owner,
|
||||
since=since,
|
||||
escalate_fields=dict(escalate_fields or {}),
|
||||
strict_fields=strict_fields,
|
||||
safe_fields=set(safe_fields or ()),
|
||||
escalate_unknown_scope=escalate_unknown_scope,
|
||||
)
|
||||
GATEWAY_METHODS[name] = desc
|
||||
return desc
|
||||
|
||||
|
||||
def resolve_required_scope(
|
||||
method: str, params: Optional[Dict[str, Any]] = None
|
||||
) -> OperatorScope:
|
||||
"""Resolve the scope required to invoke ``method`` with ``params``.
|
||||
|
||||
Default-deny: an unclassified/unknown method requires ``ADMIN`` so new
|
||||
control surface is closed until explicitly classified — the omission fails
|
||||
**closed** rather than open.
|
||||
"""
|
||||
desc = GATEWAY_METHODS.get(method)
|
||||
if desc is None:
|
||||
return OperatorScope.ADMIN
|
||||
return desc.resolve(params)
|
||||
|
||||
|
||||
# Core method classification. Registered once at import so the dispatcher can
|
||||
# consult the registry instead of scattered per-endpoint checks. Structural
|
||||
# mutations (channel control) demand ADMIN; sending as the agent needs WRITE;
|
||||
# status/read needs READ. Anything unregistered defaults to ADMIN (deny).
|
||||
def _register_core_gateway_methods() -> None:
|
||||
core: Dict[str, OperatorScope] = {
|
||||
"agent.message": OperatorScope.WRITE,
|
||||
"message": OperatorScope.WRITE,
|
||||
"session.status": OperatorScope.READ,
|
||||
"session.transcript": OperatorScope.READ,
|
||||
"approvals.resolve": OperatorScope.APPROVALS,
|
||||
"pairing.approve": OperatorScope.PAIRING,
|
||||
"pairing.revoke": OperatorScope.PAIRING,
|
||||
"channels.control": OperatorScope.ADMIN,
|
||||
"channels.pause": OperatorScope.ADMIN,
|
||||
"channels.resume": OperatorScope.ADMIN,
|
||||
"channels.reconnect": OperatorScope.ADMIN,
|
||||
}
|
||||
for name, scope in core.items():
|
||||
register_gateway_method(name, scope=scope, owner="core", replace=True)
|
||||
|
||||
|
||||
_register_core_gateway_methods()
|
||||
@@ -113,10 +113,38 @@ class OllamaAdapter(DefaultAdapter):
|
||||
return 1 # Ollama-specific threshold
|
||||
|
||||
def format_tool_result_message(self, function_name: str, tool_result: Any, tool_call_id: Optional[str] = None) -> Dict[str, Any]:
|
||||
# Ollama uses natural language format for tool results
|
||||
# Ollama uses natural language format for tool results.
|
||||
# Error results get a distinct, apology-oriented instruction so the model
|
||||
# explains the failure rather than echoing the raw error.
|
||||
is_error = False
|
||||
error_message = None
|
||||
if isinstance(tool_result, dict) and 'error' in tool_result:
|
||||
is_error = True
|
||||
error_message = tool_result.get('error', 'Unknown error')
|
||||
elif isinstance(tool_result, list) and len(tool_result) > 0:
|
||||
first_item = tool_result[0]
|
||||
if isinstance(first_item, dict) and 'error' in first_item:
|
||||
is_error = True
|
||||
error_message = first_item.get('error', 'Unknown error')
|
||||
|
||||
if is_error:
|
||||
return {
|
||||
"role": "user",
|
||||
"content": f"""The tool "{function_name}" encountered an error:
|
||||
{error_message}
|
||||
|
||||
Please provide a helpful response to the user explaining that the operation could not be completed.
|
||||
Be apologetic and suggest alternatives if possible. Do NOT repeat the raw error message.
|
||||
Give a natural, conversational response."""
|
||||
}
|
||||
|
||||
return {
|
||||
"role": "user",
|
||||
"content": f"Tool '{function_name}' returned: {tool_result}"
|
||||
"role": "user",
|
||||
"content": f"""Tool execution complete.
|
||||
Function: {function_name}
|
||||
Result: {tool_result}
|
||||
|
||||
Now provide your final answer using this result. Summarize the information naturally for the user."""
|
||||
}
|
||||
|
||||
def handle_empty_response_with_tools(self, state: Dict[str, Any]) -> bool:
|
||||
|
||||
@@ -1518,47 +1518,21 @@ Respond with ONLY a valid JSON tool call in this format:
|
||||
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
def _format_ollama_tool_result_message(self, function_name: str, tool_result: Any) -> Dict[str, str]:
|
||||
def _format_ollama_tool_result_message(self, function_name: str, tool_result: Any) -> Dict[str, Any]:
|
||||
"""
|
||||
Format tool result message for Ollama provider.
|
||||
Enhanced to instruct model to use the result for final answer.
|
||||
Handles error results with specific instructions to provide user-friendly responses.
|
||||
|
||||
Delegates to the protocol-driven ``OllamaAdapter.format_tool_result_message``
|
||||
so this provider-specific formatting lives in the adapter layer (single
|
||||
source of truth). Reuses the already-constructed ``_provider_adapter`` when
|
||||
it is the Ollama adapter, otherwise instantiates one (these call sites are
|
||||
guarded by ``_is_ollama_provider()``, so Ollama formatting is always used).
|
||||
"""
|
||||
# Check if the result is an error
|
||||
is_error = False
|
||||
error_message = None
|
||||
|
||||
if isinstance(tool_result, dict) and 'error' in tool_result:
|
||||
is_error = True
|
||||
error_message = tool_result.get('error', 'Unknown error')
|
||||
elif isinstance(tool_result, list) and len(tool_result) > 0:
|
||||
first_item = tool_result[0]
|
||||
if isinstance(first_item, dict) and 'error' in first_item:
|
||||
is_error = True
|
||||
error_message = first_item.get('error', 'Unknown error')
|
||||
|
||||
if is_error:
|
||||
# For errors, provide clear instructions to give a helpful response
|
||||
return {
|
||||
"role": "user",
|
||||
"content": f"""The tool "{function_name}" encountered an error:
|
||||
{error_message}
|
||||
|
||||
Please provide a helpful response to the user explaining that the operation could not be completed.
|
||||
Be apologetic and suggest alternatives if possible. Do NOT repeat the raw error message.
|
||||
Give a natural, conversational response."""
|
||||
}
|
||||
else:
|
||||
# For successful results, format normally
|
||||
tool_result_str = str(tool_result)
|
||||
return {
|
||||
"role": "user",
|
||||
"content": f"""Tool execution complete.
|
||||
Function: {function_name}
|
||||
Result: {tool_result_str}
|
||||
|
||||
Now provide your final answer using this result. Summarize the information naturally for the user."""
|
||||
}
|
||||
from .adapters import OllamaAdapter
|
||||
adapter = getattr(self, '_provider_adapter', None)
|
||||
if not isinstance(adapter, OllamaAdapter):
|
||||
adapter = OllamaAdapter()
|
||||
return adapter.format_tool_result_message(function_name, tool_result)
|
||||
|
||||
def _try_append_multimodal_tool_result(
|
||||
self,
|
||||
@@ -5248,7 +5222,17 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
return 4000 # Safe default
|
||||
|
||||
def _setup_event_tracking(self, events: List[Any]) -> None:
|
||||
"""Setup callback functions for tracking model usage"""
|
||||
"""Setup callback functions for tracking model usage.
|
||||
|
||||
``litellm.callbacks`` is a process-global list shared by every LLM
|
||||
instance. Overwriting it wipes callbacks registered by other concurrent
|
||||
agents, so we merge into it instead: only the callbacks *this* instance
|
||||
previously registered are removed, and unrelated instances' callbacks
|
||||
are never touched. An empty ``events`` list is a no-op.
|
||||
"""
|
||||
if not events:
|
||||
return
|
||||
|
||||
try:
|
||||
import litellm
|
||||
except ImportError:
|
||||
@@ -5267,8 +5251,19 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
for event in litellm._async_success_callback[:]:
|
||||
if type(event) in event_types:
|
||||
litellm._async_success_callback.remove(event)
|
||||
|
||||
litellm.callbacks = events
|
||||
|
||||
# Merge into the global list rather than replacing it. Only remove the
|
||||
# callbacks this instance registered on a prior call, then append the
|
||||
# current ones, preserving other instances' callbacks.
|
||||
if litellm.callbacks is None:
|
||||
litellm.callbacks = []
|
||||
for cb in getattr(self, "_registered_callbacks", []):
|
||||
if cb in litellm.callbacks:
|
||||
litellm.callbacks.remove(cb)
|
||||
for event in events:
|
||||
if event not in litellm.callbacks:
|
||||
litellm.callbacks.append(event)
|
||||
self._registered_callbacks = list(events)
|
||||
|
||||
def _track_token_usage(self, response: Dict[str, Any], model: str) -> Optional[TokenMetrics]:
|
||||
"""Extract and track token usage from LLM response."""
|
||||
|
||||
@@ -261,7 +261,8 @@ class ChromaMemoryAdapter:
|
||||
)
|
||||
|
||||
# Initialize collection
|
||||
collection_name = "memory_store"
|
||||
collection_name = kwargs.get("collection_name", "memory_store")
|
||||
self.collection_name = collection_name
|
||||
try:
|
||||
self.collection = self.client.get_collection(name=collection_name)
|
||||
except Exception:
|
||||
|
||||
@@ -246,6 +246,7 @@ class Memory(SearchMixin, MemoryCoreMixin):
|
||||
if self.use_rag and hasattr(adapter, 'collection'):
|
||||
self.chroma_col = adapter.collection
|
||||
self.chroma_client = adapter.client
|
||||
self._collection_name = getattr(adapter, 'collection_name', 'memory_store')
|
||||
|
||||
if self.use_mongodb:
|
||||
if hasattr(adapter, 'client'):
|
||||
@@ -287,6 +288,7 @@ class Memory(SearchMixin, MemoryCoreMixin):
|
||||
config["short_db"] = self.cfg.get("short_db", os.path.join(project_data, "short_term.db"))
|
||||
config["long_db"] = self.cfg.get("long_db", os.path.join(project_data, "long_term.db"))
|
||||
config["rag_db_path"] = self.cfg.get("rag_db_path", os.path.join(project_data, "chroma_db"))
|
||||
config["collection_name"] = self.cfg.get("collection_name", "memory_store")
|
||||
config["verbose"] = self.verbose
|
||||
|
||||
# Add specific configurations for different adapters
|
||||
@@ -443,7 +445,8 @@ class Memory(SearchMixin, MemoryCoreMixin):
|
||||
)
|
||||
)
|
||||
|
||||
collection_name = "memory_store"
|
||||
collection_name = self.cfg.get("collection_name", "memory_store")
|
||||
self._collection_name = collection_name
|
||||
try:
|
||||
self.chroma_col = self.chroma_client.get_collection(name=collection_name)
|
||||
self._log_verbose("Using existing ChromaDB collection")
|
||||
@@ -1150,8 +1153,24 @@ class Memory(SearchMixin, MemoryCoreMixin):
|
||||
except Exception as e:
|
||||
self._log_verbose(f"Error clearing MongoDB long-term memory: {e}", logging.ERROR)
|
||||
if self.use_rag and hasattr(self, "chroma_client"):
|
||||
self.chroma_client.reset() # entire DB
|
||||
self._init_chroma() # re-init fresh
|
||||
# Scope the reset to this instance's own collection. A full
|
||||
# ``chroma_client.reset()`` wipes *every* collection in the shared
|
||||
# persistent store, destroying other agents' long-term memory that
|
||||
# happens to live in the same directory.
|
||||
collection_name = getattr(self, "_collection_name", "memory_store")
|
||||
try:
|
||||
self.chroma_client.delete_collection(name=collection_name)
|
||||
except Exception as e:
|
||||
# Surface the failure instead of swallowing it: reopening the
|
||||
# unchanged collection below would otherwise let the caller
|
||||
# believe the reset succeeded while the long-term memories
|
||||
# remain fully available.
|
||||
self._log_verbose(
|
||||
f"Error deleting ChromaDB collection '{collection_name}': {e}",
|
||||
logging.ERROR,
|
||||
)
|
||||
raise
|
||||
self._init_chroma() # recreate only this collection
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Selective Deletion Methods
|
||||
|
||||
@@ -15,6 +15,7 @@ Design goals:
|
||||
never silently weakened.
|
||||
"""
|
||||
|
||||
import re
|
||||
import shlex
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List
|
||||
@@ -23,6 +24,36 @@ from typing import List
|
||||
# Operators that separate simple-commands within a compound command.
|
||||
_SEPARATORS = ("&&", "||", ";", "|", "&", "\n")
|
||||
|
||||
# Shell *parameter* expansion that bash resolves at runtime, invisibly to the
|
||||
# static tokenizer. ``rm${IFS}-rf${IFS}/`` tokenizes as one opaque token here
|
||||
# but bash word-splits it into ``rm -rf /`` when executed, so a broad allow
|
||||
# rule could match while a specific ``rm -rf *`` deny never does. Likewise a
|
||||
# ``${VAR}`` or bare ``$VAR`` (e.g. ``rm -rf $HOME``) expands to an unknown
|
||||
# value at runtime that could evade a value-specific deny while matching a
|
||||
# broad ``bash:*`` allow. Command substitution (``$(...)`` / backticks) is
|
||||
# deliberately *not* matched here: it is decomposed and evaluated per-op by
|
||||
# ``parse_command`` already, so a deny on the inner command still fires. The
|
||||
# pattern only accepts ``${`` or ``$`` followed by an identifier char, so
|
||||
# ``$(`` (command substitution) never matches. Only parameter expansion,
|
||||
# which cannot be statically resolved, is treated as unverifiable.
|
||||
_UNSAFE_EXPANSION_RE = re.compile(r"\$(?:\{|[A-Za-z_])")
|
||||
|
||||
|
||||
def has_unresolvable_expansion(cmd: str) -> bool:
|
||||
"""Return ``True`` if *cmd* contains parameter expansion we cannot resolve.
|
||||
|
||||
Detects ``${...}`` parameter expansion and bare ``$VAR`` references
|
||||
(including ``$IFS`` word-splitting tricks), both resolved by bash at
|
||||
runtime and invisible to a static tokenizer, so they cannot be safely
|
||||
matched against deny rules. Command substitution (``$(...)``/backticks) is
|
||||
excluded because it is decomposed per-operation elsewhere. Callers should
|
||||
escalate matching commands to ``ASK`` rather than letting a broad allow
|
||||
rule short-circuit.
|
||||
"""
|
||||
if not cmd:
|
||||
return False
|
||||
return _UNSAFE_EXPANSION_RE.search(cmd) is not None
|
||||
|
||||
# Redirection operators that truncate/overwrite or append to a file.
|
||||
# These produce an additional ``write:<path>`` sub-target.
|
||||
_WRITE_REDIRECTS = (">", ">>", ">|", "&>", "&>>")
|
||||
|
||||
@@ -5,6 +5,7 @@ Manages permission rules, persistent approvals, and integrates
|
||||
with doom loop detection.
|
||||
"""
|
||||
|
||||
import fnmatch
|
||||
import json
|
||||
import logging
|
||||
from praisonaiagents._logging import get_logger
|
||||
@@ -226,6 +227,92 @@ class PermissionManager:
|
||||
"delete_file:",
|
||||
)
|
||||
|
||||
# Read-tool prefixes whose ``<prefix>:<path>`` target is gated by the
|
||||
# secret-file default. A default coding agent can otherwise ``read`` a
|
||||
# ``.env``/private key and forward its contents to the model provider — a
|
||||
# silent secret-exfiltration path. Reads of these files default to ``ask``.
|
||||
_READ_PREFIXES = (
|
||||
"read:",
|
||||
"read_file:",
|
||||
)
|
||||
|
||||
# Basename globs that identify a secret file. Matched case-insensitively
|
||||
# against the *basename* so ``config/.env`` and ``.env`` both gate. Safe
|
||||
# example/sample files are excluded via ``_SECRET_ALLOW_PATTERNS`` below.
|
||||
_SECRET_PATTERNS = (
|
||||
".env",
|
||||
"*.env",
|
||||
".env.*",
|
||||
"*.env.*",
|
||||
"*.pem",
|
||||
"*.key",
|
||||
"id_rsa",
|
||||
"id_dsa",
|
||||
"id_ecdsa",
|
||||
"id_ed25519",
|
||||
"*.pfx",
|
||||
"*.p12",
|
||||
)
|
||||
|
||||
# Basename globs that are always safe to read even though they match a
|
||||
# secret pattern (documentation templates without real secrets).
|
||||
_SECRET_ALLOW_PATTERNS = (
|
||||
"*.env.example",
|
||||
"*.env.sample",
|
||||
"*.env.template",
|
||||
".env.example",
|
||||
".env.sample",
|
||||
".env.template",
|
||||
"*.example",
|
||||
"*.sample",
|
||||
"*.template",
|
||||
)
|
||||
|
||||
def _is_secret_read_path(self, path: str) -> bool:
|
||||
"""Return ``True`` if *path* is a secret file that should gate reads.
|
||||
|
||||
Matches the file's basename (case-insensitively) against the built-in
|
||||
secret globs, excluding safe example/sample/template files. Purely
|
||||
pattern-based (no filesystem access) so it is cheap and side-effect
|
||||
free.
|
||||
"""
|
||||
basename = os.path.basename(path.rstrip("/\\").strip()).lower()
|
||||
if not basename:
|
||||
return False
|
||||
if any(fnmatch.fnmatch(basename, p) for p in self._SECRET_ALLOW_PATTERNS):
|
||||
return False
|
||||
return any(fnmatch.fnmatch(basename, p) for p in self._SECRET_PATTERNS)
|
||||
|
||||
# Non-secret basenames used to probe whether an allow rule is a broad
|
||||
# catch-all (matches everything) rather than a secret-specific opt-in.
|
||||
_NON_SECRET_PROBES = ("main.py", "readme.md", "data.txt")
|
||||
|
||||
def _is_specific_secret_allow(self, rule: "PermissionRule") -> bool:
|
||||
"""Return ``True`` if *rule* is a secret-specific read allow (opt-in).
|
||||
|
||||
A rule opts a secret read in only when it targets secrets specifically
|
||||
(e.g. ``read:*.env``, ``read:*.pem``) — not when it is a broad catch-all
|
||||
such as ``read:*`` that also matches ordinary files. We test the rule's
|
||||
glob against a set of clearly non-secret basenames: if it matches any of
|
||||
them it is treated as broad and does *not* silently authorise secrets.
|
||||
Regex rules are conservatively treated as specific (author was explicit).
|
||||
"""
|
||||
if rule.action != PermissionAction.ALLOW:
|
||||
return False
|
||||
if getattr(rule, "is_regex", False):
|
||||
return True
|
||||
pattern = rule.pattern
|
||||
for prefix in self._READ_PREFIXES:
|
||||
if pattern.startswith(prefix):
|
||||
pattern = pattern[len(prefix):]
|
||||
break
|
||||
glob = os.path.basename(pattern.rstrip("/\\")) or pattern
|
||||
# A broad glob (``*``) matches ordinary files too → not secret-specific.
|
||||
for probe in self._NON_SECRET_PROBES:
|
||||
if fnmatch.fnmatch(probe, glob.lower()):
|
||||
return False
|
||||
return True
|
||||
|
||||
def check(self, target: str, agent_name: Optional[str] = None) -> PermissionResult:
|
||||
"""
|
||||
Check permission for a target.
|
||||
@@ -265,6 +352,41 @@ class PermissionManager:
|
||||
if boundary is not None:
|
||||
return boundary
|
||||
|
||||
# Secret-file read gate: reading ``.env``/private keys and forwarding
|
||||
# them to the model provider is a silent secret-leak path. Such reads
|
||||
# default to ``ask`` even when no rule matches. An explicit user
|
||||
# rule/approval (e.g. ``read:*.env=allow``) still overrides, so opting
|
||||
# in for a trusted workflow remains one rule.
|
||||
read_prefix = next(
|
||||
(p for p in self._READ_PREFIXES if target.startswith(p)), None
|
||||
)
|
||||
if read_prefix is not None and self._is_secret_read_path(
|
||||
target[len(read_prefix):]
|
||||
):
|
||||
flat = self._check_flat(target, agent)
|
||||
# A persistent approval or an explicit *deny* always wins (opt-in or
|
||||
# hardening). An *allow* only opts in when it comes from a rule that
|
||||
# specifically targets secrets — a broad wildcard (``read:*``) must
|
||||
# not silently authorise credential files, otherwise the gate is
|
||||
# trivially defeated by the catch-all rule most agents ship with.
|
||||
if flat.approved is not None:
|
||||
return flat
|
||||
if flat.rule is not None:
|
||||
if flat.action == PermissionAction.DENY:
|
||||
return flat
|
||||
if self._is_specific_secret_allow(flat.rule):
|
||||
return flat
|
||||
# Broad allow/ask rule: fall through to the secret ASK default.
|
||||
return PermissionResult(
|
||||
action=PermissionAction.ASK,
|
||||
target=target,
|
||||
reason=(
|
||||
"Reading a secret file requires approval "
|
||||
"(override with a secret-specific 'read' allow rule, "
|
||||
"e.g. read:*.env, to opt in)"
|
||||
),
|
||||
)
|
||||
|
||||
return self._check_flat(target, agent)
|
||||
|
||||
def _check_file_boundary(
|
||||
@@ -351,7 +473,31 @@ class PermissionManager:
|
||||
"""
|
||||
original_target = prefix + command
|
||||
try:
|
||||
from .command_parser import parse_command
|
||||
from .command_parser import parse_command, has_unresolvable_expansion
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
# Shell parameter/command expansion (``${IFS}``, ``$(...)``, backticks)
|
||||
# is resolved by bash at runtime and is invisible to the static
|
||||
# tokenizer, so a payload like ``rm${IFS}-rf${IFS}/`` could slip past a
|
||||
# specific deny rule while a broad ``bash:*`` allow matches. When such
|
||||
# expansion is present we cannot statically verify the command: an
|
||||
# explicit deny still wins, but otherwise we escalate to ASK rather than
|
||||
# letting the flat matcher optimistically ALLOW it.
|
||||
if has_unresolvable_expansion(command):
|
||||
flat = self._check_flat(original_target, agent)
|
||||
if flat.action == PermissionAction.DENY:
|
||||
return flat
|
||||
return PermissionResult(
|
||||
action=PermissionAction.ASK,
|
||||
target=original_target,
|
||||
reason=(
|
||||
"Command contains shell expansion that cannot be "
|
||||
"statically verified; requires approval"
|
||||
),
|
||||
)
|
||||
|
||||
try:
|
||||
ops = parse_command(command)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@@ -69,11 +69,14 @@ __all__ = [
|
||||
# Doctor migration protocol
|
||||
"DoctorContractProtocol",
|
||||
"Finding",
|
||||
"ConfigDiff",
|
||||
"RepairPlan",
|
||||
"get_default_registry",
|
||||
"register_rule",
|
||||
"get_rules",
|
||||
"collect_findings",
|
||||
"apply_fixes",
|
||||
"plan_fixes",
|
||||
# Capability types
|
||||
"RuntimeCapability",
|
||||
"RuntimeCapabilityMatrix",
|
||||
@@ -151,6 +154,8 @@ _LAZY_GROUPS = {
|
||||
'doctor_protocol': {
|
||||
'DoctorContractProtocol': ('praisonaiagents.runtime.doctor_protocol', 'DoctorContractProtocol'),
|
||||
'Finding': ('praisonaiagents.runtime.doctor_protocol', 'Finding'),
|
||||
'ConfigDiff': ('praisonaiagents.runtime.doctor_protocol', 'ConfigDiff'),
|
||||
'RepairPlan': ('praisonaiagents.runtime.doctor_protocol', 'RepairPlan'),
|
||||
},
|
||||
'doctor_registry': {
|
||||
'get_default_registry': ('praisonaiagents.runtime.doctor_registry', 'get_default_registry'),
|
||||
@@ -158,6 +163,7 @@ _LAZY_GROUPS = {
|
||||
'get_rules': ('praisonaiagents.runtime.doctor_registry', 'get_rules'),
|
||||
'collect_findings': ('praisonaiagents.runtime.doctor_registry', 'collect_findings'),
|
||||
'apply_fixes': ('praisonaiagents.runtime.doctor_registry', 'apply_fixes'),
|
||||
'plan_fixes': ('praisonaiagents.runtime.doctor_registry', 'plan_fixes'),
|
||||
},
|
||||
'journal': {
|
||||
'RunJournal': ('praisonaiagents.runtime.journal', 'RunJournal'),
|
||||
|
||||
@@ -21,6 +21,63 @@ class Finding:
|
||||
context: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConfigDiff:
|
||||
"""A per-rule before/after change produced by a repair."""
|
||||
|
||||
rule_id: str
|
||||
before: Dict[str, Any]
|
||||
after: Dict[str, Any]
|
||||
|
||||
def unified_diff(self) -> str:
|
||||
"""Render a unified before/after diff for previewing the change."""
|
||||
import difflib
|
||||
import json
|
||||
|
||||
before_lines = json.dumps(self.before, indent=2, sort_keys=True, default=str).splitlines()
|
||||
after_lines = json.dumps(self.after, indent=2, sort_keys=True, default=str).splitlines()
|
||||
diff = difflib.unified_diff(
|
||||
before_lines,
|
||||
after_lines,
|
||||
fromfile=f"{self.rule_id} (before)",
|
||||
tofile=f"{self.rule_id} (after)",
|
||||
lineterm="",
|
||||
)
|
||||
return "\n".join(diff)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RepairPlan:
|
||||
"""
|
||||
Result of a doctor repair pass, carrying the safety contract surface.
|
||||
|
||||
In dry-run mode nothing is written; ``config`` holds the proposed result and
|
||||
``diffs`` describes each change so a caller can preview before applying.
|
||||
"""
|
||||
|
||||
config: Dict[str, Any]
|
||||
diffs: List[ConfigDiff]
|
||||
backup_path: Optional[str] = None
|
||||
residual_findings: Optional[List[Finding]] = None
|
||||
refused: Optional[List[Finding]] = None
|
||||
applied: bool = False
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.residual_findings is None:
|
||||
self.residual_findings = []
|
||||
if self.refused is None:
|
||||
self.refused = []
|
||||
|
||||
@property
|
||||
def has_changes(self) -> bool:
|
||||
"""True if any rule proposed a change."""
|
||||
return len(self.diffs) > 0
|
||||
|
||||
def render_diffs(self) -> str:
|
||||
"""Render all per-rule unified diffs joined together."""
|
||||
return "\n".join(d.unified_diff() for d in self.diffs)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class DoctorContractProtocol(Protocol):
|
||||
"""
|
||||
|
||||
@@ -7,7 +7,12 @@ Manages built-in and plugin-provided migration rules.
|
||||
import copy
|
||||
import warnings
|
||||
from typing import Any, Dict, List, Optional
|
||||
from .doctor_protocol import DoctorContractProtocol, Finding
|
||||
from .doctor_protocol import (
|
||||
DoctorContractProtocol,
|
||||
Finding,
|
||||
ConfigDiff,
|
||||
RepairPlan,
|
||||
)
|
||||
from .builtin_rules import CliBackendMigrationRule
|
||||
|
||||
|
||||
@@ -62,18 +67,140 @@ class DoctorRulesRegistry:
|
||||
return all_findings
|
||||
|
||||
def apply_all_fixes(self, config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Apply fixes from all rules that have findings."""
|
||||
"""
|
||||
Apply fixes from all rules that have findings.
|
||||
|
||||
Backward-compatible convenience wrapper that returns only the repaired
|
||||
config dict. For the safety-rail surface (diff preview, backup, refuse,
|
||||
re-validation) use :meth:`plan_fixes`.
|
||||
"""
|
||||
return self.plan_fixes(config).config
|
||||
|
||||
def plan_fixes(
|
||||
self,
|
||||
config: Dict[str, Any],
|
||||
*,
|
||||
dry_run: bool = True,
|
||||
backup: bool = False,
|
||||
backup_path: Optional[str] = None,
|
||||
) -> RepairPlan:
|
||||
"""
|
||||
Produce a :class:`RepairPlan` describing what the doctor repair changes.
|
||||
|
||||
Safety contract:
|
||||
- **Diff preview**: every applied rule records a before/after ``ConfigDiff``.
|
||||
- **Refuse-on-unrecoverable**: a rule that raises is skipped, its blocking
|
||||
finding(s) recorded in ``refused``, and its partial change discarded.
|
||||
- **Backup**: when ``dry_run=False`` and ``backup=True`` the original
|
||||
config is snapshotted to ``backup_path`` before returning.
|
||||
- **Re-validation**: after applying, residual findings are re-collected so
|
||||
callers can report "repair left N finding(s)".
|
||||
|
||||
Args:
|
||||
config: The configuration to repair (never mutated).
|
||||
dry_run: When True (default) nothing is written to disk; the plan's
|
||||
``config`` holds the proposed result for previewing.
|
||||
backup: When True (and not dry_run), write a timestamped backup of
|
||||
the original config before applying.
|
||||
backup_path: Explicit backup destination. If omitted a timestamped
|
||||
path is derived.
|
||||
|
||||
Returns:
|
||||
A :class:`RepairPlan`.
|
||||
"""
|
||||
result = copy.deepcopy(config)
|
||||
|
||||
diffs: List[ConfigDiff] = []
|
||||
refused: List[Finding] = []
|
||||
|
||||
for rule in self.get_rules():
|
||||
try:
|
||||
findings = rule.collect_findings(result)
|
||||
if findings:
|
||||
result = rule.apply_fix(result)
|
||||
except Exception as e:
|
||||
warnings.warn(f"Error applying fix for rule '{rule.rule_id}': {e}")
|
||||
|
||||
return result
|
||||
# Refuse-on-unrecoverable: a rule that cannot even verify the
|
||||
# config is a blocking condition, not a silent skip. Record it
|
||||
# so callers see it in ``refused`` instead of an apparently
|
||||
# clean plan.
|
||||
warnings.warn(f"Refusing rule '{rule.rule_id}' (collect failed): {e}")
|
||||
refused.append(Finding(
|
||||
rule_id=rule.rule_id,
|
||||
severity="error",
|
||||
message=f"Refused to collect findings (unrecoverable): {e}",
|
||||
fix_description=None,
|
||||
context={"error": str(e)},
|
||||
))
|
||||
continue
|
||||
|
||||
if not findings:
|
||||
continue
|
||||
|
||||
# Snapshot the prior state, and hand the rule its *own* fresh copy so
|
||||
# that rules which mutate-and-return their argument (allowed by the
|
||||
# protocol) don't alias ``before`` and get their change discarded by
|
||||
# the ``fixed != before`` comparison below.
|
||||
before = copy.deepcopy(result)
|
||||
try:
|
||||
fixed = rule.apply_fix(copy.deepcopy(before))
|
||||
except Exception as e:
|
||||
# Refuse-on-unrecoverable: discard this rule's change, preserve
|
||||
# the prior state, and record the blocking finding(s).
|
||||
warnings.warn(f"Refusing fix for rule '{rule.rule_id}': {e}")
|
||||
refused.append(Finding(
|
||||
rule_id=rule.rule_id,
|
||||
severity="error",
|
||||
message=f"Refused to apply fix (unrecoverable): {e}",
|
||||
fix_description=None,
|
||||
context={"error": str(e)},
|
||||
))
|
||||
continue
|
||||
|
||||
if fixed != before:
|
||||
diffs.append(ConfigDiff(rule_id=rule.rule_id, before=before, after=copy.deepcopy(fixed)))
|
||||
result = fixed
|
||||
|
||||
residual_findings = self.collect_all_findings(result)
|
||||
|
||||
# Only report "applied" when a repair actually changed something; a
|
||||
# no-op run in apply mode must not claim a repair was performed.
|
||||
applied = (not dry_run) and bool(diffs)
|
||||
|
||||
written_backup_path: Optional[str] = None
|
||||
if not dry_run and backup and diffs:
|
||||
written_backup_path = self._write_backup(config, backup_path)
|
||||
|
||||
return RepairPlan(
|
||||
config=result,
|
||||
diffs=diffs,
|
||||
backup_path=written_backup_path,
|
||||
residual_findings=residual_findings,
|
||||
refused=refused,
|
||||
applied=applied,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _write_backup(config: Dict[str, Any], backup_path: Optional[str]) -> str:
|
||||
"""Snapshot the original config to a timestamped backup file.
|
||||
|
||||
For an auto-derived path the timestamp includes microseconds and, if a
|
||||
file still collides (concurrent runs / quick retry), a numeric suffix is
|
||||
appended so an earlier snapshot is never overwritten.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
if backup_path is None:
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
|
||||
candidate = f"doctor-config.bak-{timestamp}.json"
|
||||
counter = 1
|
||||
while os.path.exists(candidate):
|
||||
candidate = f"doctor-config.bak-{timestamp}-{counter}.json"
|
||||
counter += 1
|
||||
backup_path = candidate
|
||||
|
||||
with open(backup_path, "w", encoding="utf-8") as f:
|
||||
json.dump(config, f, indent=2, sort_keys=True, default=str)
|
||||
|
||||
return backup_path
|
||||
|
||||
def _load_builtin_rules(self) -> None:
|
||||
"""Load built-in migration rules."""
|
||||
@@ -144,4 +271,17 @@ def collect_findings(config: Dict[str, Any]) -> List[Finding]:
|
||||
|
||||
def apply_fixes(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Apply fixes from all rules in the default registry."""
|
||||
return get_default_registry().apply_all_fixes(config)
|
||||
return get_default_registry().apply_all_fixes(config)
|
||||
|
||||
|
||||
def plan_fixes(
|
||||
config: Dict[str, Any],
|
||||
*,
|
||||
dry_run: bool = True,
|
||||
backup: bool = False,
|
||||
backup_path: Optional[str] = None,
|
||||
) -> RepairPlan:
|
||||
"""Produce a RepairPlan from all rules in the default registry."""
|
||||
return get_default_registry().plan_fixes(
|
||||
config, dry_run=dry_run, backup=backup, backup_path=backup_path
|
||||
)
|
||||
@@ -17,6 +17,7 @@ Usage:
|
||||
Default storage: ~/.praisonai/config.yaml (under ``schedules`` key)
|
||||
"""
|
||||
|
||||
import threading
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -35,6 +36,50 @@ if TYPE_CHECKING:
|
||||
|
||||
_module_cache = {}
|
||||
|
||||
# ── canonical default store ──────────────────────────────────────────────────
|
||||
# Every first-class consumer (agent tools, gateway tick, host bridge) resolves
|
||||
# its store through ``get_default_store()`` so the writer and reader can never
|
||||
# drift onto different backends (see issue #3264). ``set_default_store()``
|
||||
# overrides the singleton process-wide.
|
||||
_default_store = None
|
||||
_default_store_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_default_store():
|
||||
"""Return the process-wide canonical schedule store.
|
||||
|
||||
Lazily constructs a :class:`ConfigYamlScheduleStore` (persisting to
|
||||
``~/.praisonai/config.yaml``) on first use, migrating any pre-existing
|
||||
``jobs.json`` data. All consumers share this single instance so a job
|
||||
authored on any surface is polled by the gateway ticker.
|
||||
"""
|
||||
global _default_store
|
||||
with _default_store_lock:
|
||||
if _default_store is None:
|
||||
from .config_store import ConfigYamlScheduleStore
|
||||
store = ConfigYamlScheduleStore()
|
||||
try:
|
||||
store.migrate_from_json()
|
||||
except Exception as e:
|
||||
import logging
|
||||
logging.getLogger(__name__).debug(
|
||||
"jobs.json migration skipped: %s", e,
|
||||
)
|
||||
_default_store = store
|
||||
return _default_store
|
||||
|
||||
|
||||
def set_default_store(store):
|
||||
"""Override the process-wide canonical schedule store.
|
||||
|
||||
A deployment that deliberately swaps the backend calls this once at
|
||||
startup; the override is then honoured by the agent tools, the gateway
|
||||
tick loop, and the host-integration bridge alike.
|
||||
"""
|
||||
global _default_store
|
||||
with _default_store_lock:
|
||||
_default_store = store
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
"""Lazy load scheduler components."""
|
||||
@@ -131,4 +176,6 @@ __all__ = [
|
||||
"Suggestion",
|
||||
"SuggestionStore",
|
||||
"MAX_PENDING_CAP",
|
||||
"get_default_store",
|
||||
"set_default_store",
|
||||
]
|
||||
@@ -6,18 +6,21 @@ the ``schedules`` top-level key. Thread-safe, preserves all
|
||||
other config.yaml content.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
from praisonaiagents._logging import get_logger
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from typing import Dict, List, Optional
|
||||
from typing import Dict, Iterator, List, Optional
|
||||
|
||||
from .models import ScheduleJob, RunRecord
|
||||
from .due import is_due as _is_due
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_HISTORY_FILE = "run_history.yaml"
|
||||
_LOCK_FILE = "config.schedules.lock"
|
||||
_MAX_HISTORY = 200
|
||||
|
||||
class ConfigYamlScheduleStore:
|
||||
@@ -35,10 +38,15 @@ class ConfigYamlScheduleStore:
|
||||
config_path = str(get_data_dir() / "config.yaml")
|
||||
self._path = config_path
|
||||
self._history_path = os.path.join(os.path.dirname(config_path), _HISTORY_FILE)
|
||||
self._lock_path = os.path.join(os.path.dirname(config_path), _LOCK_FILE)
|
||||
self._max_history = max_history
|
||||
self._lock = threading.RLock()
|
||||
self._jobs: Dict[str, ScheduleJob] = {}
|
||||
self._history: List[RunRecord] = []
|
||||
# In-memory record of leases we currently hold, so ``complete`` can
|
||||
# release them. Cross-process leases live in the persisted job
|
||||
# (``lease_until`` / ``lease_owner``) which is the source of truth.
|
||||
self._held_leases: Dict[str, str] = {}
|
||||
self._load()
|
||||
self._load_history()
|
||||
|
||||
@@ -46,7 +54,11 @@ class ConfigYamlScheduleStore:
|
||||
|
||||
def add(self, job: ScheduleJob) -> None:
|
||||
"""Add a job. Raises ``ValueError`` if id already exists."""
|
||||
with self._lock:
|
||||
with self._lock, self._file_lock():
|
||||
# Reload under the cross-process lock so we don't overwrite lease
|
||||
# / last_run_at written by a concurrent ``claim_due`` in another
|
||||
# process before applying our change.
|
||||
self._reload_locked()
|
||||
if job.id in self._jobs:
|
||||
raise ValueError(f"Job '{job.id}' already exists")
|
||||
self._jobs[job.id] = job
|
||||
@@ -71,12 +83,22 @@ class ConfigYamlScheduleStore:
|
||||
return jobs
|
||||
|
||||
def update(self, job: ScheduleJob) -> None:
|
||||
with self._lock:
|
||||
with self._lock, self._file_lock():
|
||||
# Reload latest on-disk state, then preserve any active lease held
|
||||
# for this job (written by a concurrent ``claim_due``) so a plain
|
||||
# ``update`` from a stale in-memory copy cannot silently drop it.
|
||||
self._reload_locked()
|
||||
existing = self._jobs.get(job.id)
|
||||
if existing is not None:
|
||||
if not getattr(job, "_lease_until", 0.0):
|
||||
job._lease_until = getattr(existing, "_lease_until", 0.0) or 0.0
|
||||
job._lease_owner = getattr(existing, "_lease_owner", None)
|
||||
self._jobs[job.id] = job
|
||||
self._save()
|
||||
|
||||
def remove(self, job_id: str) -> bool:
|
||||
with self._lock:
|
||||
with self._lock, self._file_lock():
|
||||
self._reload_locked()
|
||||
if job_id in self._jobs:
|
||||
del self._jobs[job_id]
|
||||
self._save()
|
||||
@@ -84,7 +106,8 @@ class ConfigYamlScheduleStore:
|
||||
return False
|
||||
|
||||
def remove_by_name(self, name: str) -> bool:
|
||||
with self._lock:
|
||||
with self._lock, self._file_lock():
|
||||
self._reload_locked()
|
||||
for jid, job in list(self._jobs.items()):
|
||||
if job.name == name:
|
||||
del self._jobs[jid]
|
||||
@@ -92,6 +115,142 @@ class ConfigYamlScheduleStore:
|
||||
return True
|
||||
return False
|
||||
|
||||
# ── atomic claim / lease ──────────────────────────────────────────
|
||||
|
||||
def claim_due(
|
||||
self,
|
||||
now: float,
|
||||
owner_id: str,
|
||||
lease_seconds: float = 300.0,
|
||||
) -> List[ScheduleJob]:
|
||||
"""Atomically claim due jobs; return only those won by ``owner_id``.
|
||||
|
||||
Under a cross-process OS advisory file lock we re-read the on-disk
|
||||
state (so we see claims made by other processes), then for each enabled
|
||||
job that is due and not already leased by someone else we:
|
||||
|
||||
* advance ``last_run_at`` to ``now`` (pre-advancing the schedule so a
|
||||
later poll no longer sees it as due — at-least-once → at-most-once),
|
||||
* take a lease (``lease_until = now + lease_seconds``, ``lease_owner``),
|
||||
persisting both in the same atomic write,
|
||||
* remove one-shot jobs (``delete_after_run``) immediately so no other
|
||||
ticker can pick them up.
|
||||
|
||||
A crashed run leaves a lease that expires after ``lease_seconds``; the
|
||||
job then becomes due again and is retried. Losers of the race simply do
|
||||
not see the job in their returned list.
|
||||
"""
|
||||
claimed: List[ScheduleJob] = []
|
||||
with self._lock, self._file_lock():
|
||||
# Re-read from disk so we observe cross-process claims/leases.
|
||||
self._reload_locked()
|
||||
changed = False
|
||||
for job in list(self._jobs.values()):
|
||||
if not job.enabled:
|
||||
continue
|
||||
lease_until = getattr(job, "_lease_until", 0.0) or 0.0
|
||||
lease_owner = getattr(job, "_lease_owner", None)
|
||||
# An unexpired lease held by another owner blocks the claim.
|
||||
if lease_until > now and lease_owner != owner_id:
|
||||
continue
|
||||
if not _is_due(job, now):
|
||||
continue
|
||||
# Win the claim: pre-advance + lease atomically.
|
||||
job.last_run_at = now
|
||||
job._lease_until = now + lease_seconds
|
||||
job._lease_owner = owner_id
|
||||
self._held_leases[job.id] = owner_id
|
||||
claimed.append(job)
|
||||
changed = True
|
||||
if job.delete_after_run:
|
||||
# One-shot: remove now so no competitor re-claims it.
|
||||
del self._jobs[job.id]
|
||||
if changed:
|
||||
if not self._save():
|
||||
# The lease/last_run_at advance was NOT persisted (full
|
||||
# disk, permissions, failed replace). Returning these jobs
|
||||
# would let the runner execute them while the next poll —
|
||||
# reloading the unchanged file — sees them as due again and
|
||||
# fires a duplicate. Drop the claim so it is retried cleanly.
|
||||
for job in claimed:
|
||||
self._held_leases.pop(job.id, None)
|
||||
return []
|
||||
return claimed
|
||||
|
||||
def complete(self, job_id: str, owner_id: str) -> None:
|
||||
"""Release the lease for ``job_id`` if held by ``owner_id`` (idempotent)."""
|
||||
with self._lock, self._file_lock():
|
||||
self._reload_locked()
|
||||
self._held_leases.pop(job_id, None)
|
||||
job = self._jobs.get(job_id)
|
||||
if job is None:
|
||||
return
|
||||
if getattr(job, "_lease_owner", None) != owner_id:
|
||||
return
|
||||
job._lease_until = 0.0
|
||||
job._lease_owner = None
|
||||
self._save()
|
||||
|
||||
# ── cross-process lock ────────────────────────────────────────────
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _file_lock(self) -> Iterator[None]:
|
||||
"""Hold an OS advisory lock on a sidecar file for the block's duration.
|
||||
|
||||
Uses ``fcntl`` on POSIX and ``msvcrt`` on Windows. If neither is
|
||||
available (or locking fails) we degrade to the in-process
|
||||
``threading`` lock already held by callers — correctness within a
|
||||
single process is preserved, only cross-process atomicity is lost.
|
||||
"""
|
||||
lock_dir = os.path.dirname(self._lock_path)
|
||||
if lock_dir:
|
||||
os.makedirs(lock_dir, exist_ok=True)
|
||||
fh = None
|
||||
locked = False
|
||||
try:
|
||||
fh = open(self._lock_path, "a+")
|
||||
try:
|
||||
import fcntl # type: ignore[import-not-found]
|
||||
fcntl.flock(fh.fileno(), fcntl.LOCK_EX)
|
||||
locked = True
|
||||
except ImportError:
|
||||
try:
|
||||
import msvcrt # type: ignore[import-not-found]
|
||||
fh.seek(0)
|
||||
msvcrt.locking(fh.fileno(), msvcrt.LK_LOCK, 1)
|
||||
locked = True
|
||||
except Exception as e: # pragma: no cover - platform dependent
|
||||
logger.debug("No OS file lock available (%s); using thread lock only", e)
|
||||
except Exception as e: # pragma: no cover - defensive
|
||||
logger.debug("Failed to acquire file lock (%s); using thread lock only", e)
|
||||
yield
|
||||
finally:
|
||||
if fh is not None:
|
||||
try:
|
||||
if locked:
|
||||
try:
|
||||
import fcntl # type: ignore[import-not-found]
|
||||
fcntl.flock(fh.fileno(), fcntl.LOCK_UN)
|
||||
except ImportError:
|
||||
try:
|
||||
import msvcrt # type: ignore[import-not-found]
|
||||
fh.seek(0)
|
||||
msvcrt.locking(fh.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
except Exception: # pragma: no cover - defensive
|
||||
pass
|
||||
fh.close()
|
||||
|
||||
def _reload_locked(self) -> None:
|
||||
"""Re-read jobs from disk (caller must hold both thread + file lock).
|
||||
|
||||
Ensures a claim decision is based on the latest cross-process state so
|
||||
a job already claimed/leased by another process is observed.
|
||||
"""
|
||||
self._jobs = {}
|
||||
self._load()
|
||||
|
||||
# ── execution history ─────────────────────────────────────────────
|
||||
|
||||
def log_run(
|
||||
@@ -154,10 +313,13 @@ class ConfigYamlScheduleStore:
|
||||
except Exception as e:
|
||||
logger.warning("Failed to load schedules from %s: %s", self._path, e)
|
||||
|
||||
def _save(self) -> None:
|
||||
def _save(self) -> bool:
|
||||
"""Write schedule jobs into config.yaml's ``schedules`` key.
|
||||
|
||||
Preserves all other top-level keys (agents, server, etc.).
|
||||
|
||||
Returns ``True`` if the write succeeded, ``False`` otherwise so
|
||||
callers (e.g. ``claim_due``) can avoid acting on unpersisted state.
|
||||
"""
|
||||
try:
|
||||
import yaml
|
||||
@@ -180,8 +342,10 @@ class ConfigYamlScheduleStore:
|
||||
with open(tmp, "w") as f:
|
||||
yaml.dump(data, f, default_flow_style=False, sort_keys=False)
|
||||
os.replace(tmp, self._path)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning("Failed to save schedules to %s: %s", self._path, e)
|
||||
return False
|
||||
|
||||
def _load_history(self) -> None:
|
||||
"""Load execution history from run_history.yaml."""
|
||||
|
||||
@@ -164,7 +164,15 @@ class FileScheduleStore:
|
||||
# One-shot: remove now so no competitor re-claims it.
|
||||
del self._jobs[job.id]
|
||||
if changed:
|
||||
self._save()
|
||||
if not self._save():
|
||||
# The lease/last_run_at advance was NOT persisted (full
|
||||
# disk, permissions, failed replace). Returning these jobs
|
||||
# would let the runner execute them while the next poll —
|
||||
# reloading the unchanged file — sees them as due again and
|
||||
# fires a duplicate. Drop the claim so it is retried cleanly.
|
||||
for job in claimed:
|
||||
self._held_leases.pop(job.id, None)
|
||||
return []
|
||||
return claimed
|
||||
|
||||
def complete(self, job_id: str, owner_id: str) -> None:
|
||||
@@ -295,7 +303,12 @@ class FileScheduleStore:
|
||||
except Exception as e:
|
||||
logger.warning("Failed to load schedule store from %s: %s", self._path, e)
|
||||
|
||||
def _save(self) -> None:
|
||||
def _save(self) -> bool:
|
||||
"""Persist jobs to disk.
|
||||
|
||||
Returns ``True`` if the write succeeded, ``False`` otherwise so
|
||||
callers (e.g. ``claim_due``) can avoid acting on unpersisted state.
|
||||
"""
|
||||
try:
|
||||
os.makedirs(self._dir, exist_ok=True)
|
||||
data = [j.to_dict() for j in self._jobs.values()]
|
||||
@@ -303,8 +316,10 @@ class FileScheduleStore:
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
os.replace(tmp, self._path)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning("Failed to save schedule store to %s: %s", self._path, e)
|
||||
return False
|
||||
|
||||
def _load_history(self) -> None:
|
||||
"""Load execution history from history.json."""
|
||||
|
||||
@@ -153,6 +153,12 @@ def __getattr__(name: str):
|
||||
_module_cache[name] = clear_session_context
|
||||
return clear_session_context
|
||||
|
||||
# Workspace-scoped default session identity (Issue #3154)
|
||||
if name == "workspace_id":
|
||||
from .workspace import workspace_id
|
||||
_module_cache[name] = workspace_id
|
||||
return workspace_id
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
@@ -181,4 +187,5 @@ __all__ = [
|
||||
"set_session_context",
|
||||
"get_session_context",
|
||||
"clear_session_context",
|
||||
"workspace_id",
|
||||
]
|
||||
@@ -286,7 +286,21 @@ class Session:
|
||||
"""
|
||||
if self.is_remote:
|
||||
return []
|
||||
|
||||
|
||||
# G-2 FIX: mirror the save path priority — read SessionStore first so
|
||||
# history persisted by _save_agent_chat_histories() is actually found.
|
||||
session_store = None
|
||||
try:
|
||||
from . import get_default_session_store
|
||||
session_store = get_default_session_store()
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
if session_store is not None and hasattr(session_store, "get_chat_history"):
|
||||
messages = session_store.get_chat_history(f"{self.session_id}_{agent_key}")
|
||||
if messages:
|
||||
return messages
|
||||
|
||||
results = self.memory.search_short_term(
|
||||
query="Agent chat history for",
|
||||
limit=10
|
||||
@@ -306,7 +320,51 @@ class Session:
|
||||
"""Restore all agent chat histories from memory."""
|
||||
if self.is_remote:
|
||||
return
|
||||
|
||||
|
||||
# G-2 FIX: read SessionStore first (where save_state now writes), then
|
||||
# fall back to Memory for backward compatibility.
|
||||
session_store = None
|
||||
try:
|
||||
from . import get_default_session_store
|
||||
session_store = get_default_session_store()
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
if session_store is not None and hasattr(session_store, "get_chat_history"):
|
||||
prefix = f"{self.session_id}_"
|
||||
list_sessions = getattr(session_store, "list_sessions", None)
|
||||
if callable(list_sessions):
|
||||
for entry in list_sessions():
|
||||
if isinstance(entry, dict):
|
||||
stored_id = entry.get("session_id")
|
||||
# Prefer the explicit parent/agent_key tags written on
|
||||
# save; this disambiguates overlapping composite ids
|
||||
# (e.g. "chat"+"support_agent" vs "chat_support"+"agent").
|
||||
tagged_parent = entry.get("parent_session_id")
|
||||
tagged_key = entry.get("agent_key")
|
||||
if tagged_parent == self.session_id and tagged_key:
|
||||
agent_key = tagged_key
|
||||
elif (
|
||||
tagged_parent is None
|
||||
and isinstance(stored_id, str)
|
||||
and stored_id.startswith(prefix)
|
||||
):
|
||||
# Legacy sessions saved before the tag existed.
|
||||
agent_key = stored_id[len(prefix):]
|
||||
else:
|
||||
continue
|
||||
else:
|
||||
stored_id = entry
|
||||
if not (isinstance(stored_id, str) and stored_id.startswith(prefix)):
|
||||
continue
|
||||
agent_key = stored_id[len(prefix):]
|
||||
messages = session_store.get_chat_history(stored_id)
|
||||
if agent_key and messages:
|
||||
self._agents[agent_key] = {
|
||||
"agent": None,
|
||||
"chat_history": messages
|
||||
}
|
||||
|
||||
results = self.memory.search_short_term(
|
||||
query="Agent chat history for",
|
||||
limit=50
|
||||
@@ -319,7 +377,7 @@ class Session:
|
||||
agent_key = metadata.get("agent_key")
|
||||
chat_history = metadata.get("chat_history", [])
|
||||
|
||||
if agent_key and chat_history:
|
||||
if agent_key and chat_history and agent_key not in self._agents:
|
||||
self._agents[agent_key] = {
|
||||
"agent": None,
|
||||
"chat_history": chat_history
|
||||
@@ -369,6 +427,17 @@ class Session:
|
||||
logging.debug(f"No chat history to persist for session {session_id}")
|
||||
elif hasattr(session_store, "set_chat_history"):
|
||||
session_store.set_chat_history(session_id, messages)
|
||||
# Tag the child session so bulk restore can resolve the
|
||||
# exact agent_key without ambiguous composite-key prefix
|
||||
# parsing (e.g. "chat" + "support_agent" vs "chat_support"
|
||||
# + "agent" both collapse to "chat_support_agent").
|
||||
update_meta = getattr(session_store, "update_session_metadata", None)
|
||||
if callable(update_meta):
|
||||
update_meta(
|
||||
session_id,
|
||||
parent_session_id=self.session_id,
|
||||
agent_key=agent_key,
|
||||
)
|
||||
else:
|
||||
# Fallback to add_message - may create duplicates on repeated calls
|
||||
logging.warning(
|
||||
|
||||
@@ -1016,6 +1016,9 @@ class DefaultSessionStore:
|
||||
# sessions from sub-agent/forked children (Issue #2655).
|
||||
"parent_id": data.get("parent_id") or (data.get("metadata") or {}).get("parent_id"),
|
||||
"parent_session_id": data.get("parent_session_id") or (data.get("metadata") or {}).get("parent_session_id"),
|
||||
# Surface the agent_key tag written by Session._save_agent_chat_histories
|
||||
# so bulk restore resolves it exactly (no prefix-parse ambiguity).
|
||||
"agent_key": data.get("agent_key") or (data.get("metadata") or {}).get("agent_key"),
|
||||
"created_at": data.get("created_at"),
|
||||
"updated_at": data.get("updated_at"),
|
||||
"message_count": len(data.get("messages", [])),
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Stable workspace/project identity for default session scoping.
|
||||
|
||||
Used by the core ``Agent`` to derive a default (auto) session id that is
|
||||
workspace-aware, so two same-named agents in different projects do not silently
|
||||
share conversation history (Issue #3154).
|
||||
|
||||
Resolution order for :func:`workspace_id`:
|
||||
|
||||
1. Git identity — the repository's root-commit sha (stable across clones,
|
||||
independent of local path), when the working directory is inside a git repo.
|
||||
2. The absolute, resolved current working directory.
|
||||
3. ``"global"`` as a last resort.
|
||||
|
||||
The returned value is an opaque, stable string; callers hash it together with
|
||||
the agent name to form the session id.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from functools import lru_cache
|
||||
|
||||
__all__ = ["workspace_id"]
|
||||
|
||||
|
||||
def _git_root_commit(cwd: str) -> str | None:
|
||||
"""Return the repo's first (root) commit sha, or ``None`` if unavailable.
|
||||
|
||||
The root commit is stable across clones and independent of the checkout
|
||||
path, making it a good project identity. Falls back silently when git is
|
||||
absent or ``cwd`` is not a repository.
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "rev-list", "--max-parents=0", "HEAD"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=2,
|
||||
check=False,
|
||||
cwd=cwd,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
if result.returncode != 0:
|
||||
return None
|
||||
# A repo may report multiple root commits; the last line is the earliest.
|
||||
lines = [ln.strip() for ln in result.stdout.splitlines() if ln.strip()]
|
||||
if not lines:
|
||||
return None
|
||||
return lines[-1]
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _resolve(cwd: str) -> str:
|
||||
root = _git_root_commit(cwd)
|
||||
if root:
|
||||
return f"git:{root}"
|
||||
return f"dir:{cwd}"
|
||||
|
||||
|
||||
def workspace_id() -> str:
|
||||
"""Return a stable identity for the current workspace/project.
|
||||
|
||||
Prefers a git root-commit identity, falls back to the resolved cwd, then
|
||||
``"global"``. Results are cached per working directory, so a long-lived
|
||||
process that changes directory (CLI, notebook, test suite) resolves the
|
||||
correct identity for each workspace while avoiding repeated ``git`` calls
|
||||
for the same directory.
|
||||
"""
|
||||
try:
|
||||
cwd = os.path.realpath(os.getcwd())
|
||||
except OSError:
|
||||
return "global"
|
||||
return _resolve(cwd)
|
||||
@@ -15,6 +15,7 @@ from .call_executor import (
|
||||
ToolTimeoutError, ToolCancelledError,
|
||||
)
|
||||
from .registry import get_registry, register_tool, get_tool, add_tool, has_tool, remove_tool, list_tools, list_available_tools, list_tools_with_allowed_filter, list_tools_with_hermes_filter, ToolRegistry
|
||||
from .resolver import resolve_tool_name, resolve_tool_names, ToolResolutionError
|
||||
from .tools import Tools
|
||||
|
||||
# Export Injected type directly for easy access
|
||||
@@ -409,6 +410,7 @@ __all__ = list(TOOL_MAPPINGS.keys()) + [
|
||||
'ToolTimeoutError', 'ToolCancelledError',
|
||||
'get_registry', 'register_tool', 'get_tool', 'add_tool', 'has_tool', 'remove_tool',
|
||||
'list_tools', 'list_available_tools', 'list_tools_with_allowed_filter', 'list_tools_with_hermes_filter', 'ToolRegistry',
|
||||
'resolve_tool_name', 'resolve_tool_names', 'ToolResolutionError',
|
||||
'ToolProxy', 'build_tool_namespace',
|
||||
'Tools',
|
||||
# Validation and retry protocols
|
||||
|
||||
@@ -8,15 +8,26 @@ Resolution order (first match wins):
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import difflib
|
||||
import importlib.util
|
||||
import logging
|
||||
from typing import Any, List, Optional
|
||||
import os
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_praisonai_tools_available: Optional[bool] = None
|
||||
|
||||
|
||||
class ToolResolutionError(ValueError):
|
||||
"""Raised in strict mode when one or more tool names cannot be resolved."""
|
||||
|
||||
def __init__(self, unknown: List[str], suggestions: Dict[str, List[str]]):
|
||||
self.unknown = unknown
|
||||
self.suggestions = suggestions
|
||||
super().__init__(_format_unknown(unknown, suggestions))
|
||||
|
||||
|
||||
def resolve_tool_name(name: str) -> Optional[Any]:
|
||||
"""Resolve a single tool name to a callable or tool instance."""
|
||||
# 1. Registry
|
||||
@@ -63,13 +74,92 @@ def resolve_tool_name(name: str) -> Optional[Any]:
|
||||
return None
|
||||
|
||||
|
||||
def resolve_tool_names(names: List[str]) -> List[Any]:
|
||||
"""Resolve tool name strings to callables/instances."""
|
||||
resolved = []
|
||||
def _available_tool_names() -> List[str]:
|
||||
"""Best-effort catalogue of known tool names for suggestions."""
|
||||
names: set = set()
|
||||
|
||||
try:
|
||||
from .registry import get_registry
|
||||
|
||||
names.update(get_registry().list_tools())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
from . import TOOL_MAPPINGS
|
||||
|
||||
names.update(TOOL_MAPPINGS.keys())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
global _praisonai_tools_available
|
||||
if _praisonai_tools_available is None:
|
||||
_praisonai_tools_available = importlib.util.find_spec("praisonai_tools") is not None
|
||||
if _praisonai_tools_available:
|
||||
try:
|
||||
import praisonai_tools
|
||||
|
||||
names.update(getattr(praisonai_tools, "__all__", []) or [])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return sorted(names)
|
||||
|
||||
|
||||
def _closest_names(name: str, limit: int = 3) -> List[str]:
|
||||
"""Return the closest known tool names to ``name``."""
|
||||
return difflib.get_close_matches(name, _available_tool_names(), n=limit, cutoff=0.6)
|
||||
|
||||
|
||||
def _format_unknown(unknown: List[str], suggestions: Dict[str, List[str]]) -> str:
|
||||
parts = []
|
||||
for name in unknown:
|
||||
near = suggestions.get(name) or []
|
||||
if near:
|
||||
hint = " Did you mean {}?".format(" or ".join(repr(s) for s in near))
|
||||
else:
|
||||
hint = ""
|
||||
parts.append("Unknown tool {!r}.{} Run 'praisonai tools list'.".format(name, hint))
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
def _default_report(unknown: List[str], suggestions: Dict[str, List[str]]) -> None:
|
||||
"""User-visible (not log-only) diagnostic for unresolved tool names."""
|
||||
logger.warning("%s", _format_unknown(unknown, suggestions))
|
||||
|
||||
|
||||
def resolve_tool_names(
|
||||
names: List[str],
|
||||
*,
|
||||
strict: Optional[bool] = None,
|
||||
on_unknown: Optional[Callable[[List[str], Dict[str, List[str]]], None]] = None,
|
||||
) -> List[Any]:
|
||||
"""Resolve tool name strings to callables/instances.
|
||||
|
||||
Args:
|
||||
names: Tool name strings to resolve.
|
||||
strict: If True, raise ``ToolResolutionError`` when any name is unknown.
|
||||
Defaults to the ``PRAISONAI_STRICT_TOOLS`` environment variable
|
||||
(falsey by default for backward compatibility).
|
||||
on_unknown: Optional callback ``(unknown, suggestions)`` invoked for
|
||||
unresolved names in non-strict mode instead of the default report.
|
||||
"""
|
||||
if strict is None:
|
||||
strict = os.getenv("PRAISONAI_STRICT_TOOLS", "").strip().lower() in ("1", "true", "yes")
|
||||
|
||||
resolved: List[Any] = []
|
||||
unknown: List[str] = []
|
||||
for name in names:
|
||||
tool = resolve_tool_name(name)
|
||||
if tool is not None:
|
||||
resolved.append(tool)
|
||||
else:
|
||||
logger.warning("Tool %r not found (registry, TOOL_MAPPINGS, praisonai-tools)", name)
|
||||
unknown.append(name)
|
||||
|
||||
if unknown:
|
||||
suggestions = {name: _closest_names(name) for name in unknown}
|
||||
if strict:
|
||||
raise ToolResolutionError(unknown=unknown, suggestions=suggestions)
|
||||
(on_unknown or _default_report)(unknown, suggestions)
|
||||
|
||||
return resolved
|
||||
@@ -33,24 +33,36 @@ def set_store(store):
|
||||
When PraisonAIUI is running, it calls this at startup to redirect
|
||||
all agent schedule_add/list/remove operations to the unified
|
||||
``config.yaml`` instead of the default ``jobs.json``.
|
||||
|
||||
This also repoints the core canonical default store so the gateway
|
||||
tick loop and host bridge read/write the same backend (issue #3264).
|
||||
"""
|
||||
global _store_instance
|
||||
with _store_instance_lock:
|
||||
_store_instance = store
|
||||
try:
|
||||
from ..scheduler import set_default_store
|
||||
set_default_store(store)
|
||||
except Exception as e:
|
||||
# Repointing the canonical default is best-effort: the local
|
||||
# ``_store_instance`` above is already authoritative for the tools.
|
||||
# Log (don't silently pass) so any reader/writer drift is diagnosable.
|
||||
logger.warning("Could not repoint canonical schedule store: %s", e)
|
||||
|
||||
def _get_store():
|
||||
"""Return (or create) the global schedule store.
|
||||
|
||||
Uses ``ConfigYamlScheduleStore`` (config.yaml) by default.
|
||||
Automatically migrates any existing ``jobs.json`` data on first use.
|
||||
Resolves the canonical default store shared by the gateway tick loop
|
||||
and host bridge (``scheduler.get_default_store()``) so a job authored
|
||||
by the agent is polled by the ticker. An explicit ``set_store()``
|
||||
override still takes precedence.
|
||||
"""
|
||||
global _store_instance
|
||||
with _store_instance_lock:
|
||||
if _store_instance is None:
|
||||
from ..scheduler.config_store import ConfigYamlScheduleStore
|
||||
_store_instance = ConfigYamlScheduleStore()
|
||||
_store_instance.migrate_from_json()
|
||||
return _store_instance
|
||||
if _store_instance is not None:
|
||||
return _store_instance
|
||||
from ..scheduler import get_default_store
|
||||
return get_default_store()
|
||||
|
||||
# ── tools ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
Loaded 100 of 264 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user