Compare commits

...
5 Commits
Author SHA1 Message Date
Mervin Praison e2e9c3f9ee Release v2.3.81 2025-12-23 08:11:31 +00:00
Mervin Praison cbe8beeca6 Release v2.3.80 2025-12-23 07:54:07 +00:00
Mervin Praison f7b66a52b7 Release v2.3.79 2025-12-23 07:27:35 +00:00
Mervin Praison 6dbf194312 Release v2.3.78 2025-12-23 06:46:15 +00:00
Mervin Praison 335e8d7e47 Release v2.3.77 2025-12-23 06:28:26 +00:00
18 changed files with 1183 additions and 208 deletions

No files matched your search

+6
View File
@@ -278,6 +278,12 @@ npm install praisonai
| ↳ Git Integration | [Example](examples/python/cli/git_integration_example.py) | [📖](https://docs.praison.ai/docs/cli/git-integration) |
| ↳ Sandbox Execution | [Example](examples/python/cli/sandbox_execution_example.py) | [📖](https://docs.praison.ai/docs/cli/sandbox-execution) |
| ↳ CLI Compare | [Example](examples/compare/cli_compare_basic.py) | [📖](https://docs.praison.ai/docs/cli/compare) |
| ↳ Profile/Benchmark | [Example](#profile-benchmark) | [📖](https://docs.praison.ai/docs/cli/profile) |
| ↳ Auto Mode | [Example](#auto-mode) | [📖](https://docs.praison.ai/docs/cli/auto) |
| ↳ Init | [Example](#init) | [📖](https://docs.praison.ai/docs/cli/init) |
| ↳ File Input | [Example](#file-input) | [📖](https://docs.praison.ai/docs/cli/file-input) |
| ↳ Final Agent | [Example](#final-agent) | [📖](https://docs.praison.ai/docs/cli/final-agent) |
| ↳ Max Tokens | [Example](#max-tokens) | [📖](https://docs.praison.ai/docs/cli/max-tokens) |
| **🧪 Evaluation** | | |
| ↳ Accuracy Evaluation | [Example](examples/eval/accuracy_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
| ↳ Performance Evaluation | [Example](examples/eval/performance_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
+1 -1
View File
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
# Install Python packages (using latest versions)
RUN pip install --no-cache-dir \
praisonai_tools \
"praisonai>=2.3.76" \
"praisonai>=2.3.81" \
"praisonai[chat]" \
"embedchain[github,youtube]"
+1 -1
View File
@@ -20,7 +20,7 @@ RUN mkdir -p /root/.praison
# Install Python packages (using latest versions)
RUN pip install --no-cache-dir \
praisonai_tools \
"praisonai>=2.3.76" \
"praisonai>=2.3.81" \
"praisonai[ui]" \
"praisonai[chat]" \
"praisonai[realtime]" \
+1 -1
View File
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
# Install Python packages (using latest versions)
RUN pip install --no-cache-dir \
praisonai_tools \
"praisonai>=2.3.76" \
"praisonai>=2.3.81" \
"praisonai[ui]" \
"praisonai[crewai]"
+6
View File
@@ -278,6 +278,12 @@ npm install praisonai
| ↳ Git Integration | [Example](examples/python/cli/git_integration_example.py) | [📖](https://docs.praison.ai/docs/cli/git-integration) |
| ↳ Sandbox Execution | [Example](examples/python/cli/sandbox_execution_example.py) | [📖](https://docs.praison.ai/docs/cli/sandbox-execution) |
| ↳ CLI Compare | [Example](examples/compare/cli_compare_basic.py) | [📖](https://docs.praison.ai/docs/cli/compare) |
| ↳ Profile/Benchmark | [Example](#profile-benchmark) | [📖](https://docs.praison.ai/docs/cli/profile) |
| ↳ Auto Mode | [Example](#auto-mode) | [📖](https://docs.praison.ai/docs/cli/auto) |
| ↳ Init | [Example](#init) | [📖](https://docs.praison.ai/docs/cli/init) |
| ↳ File Input | [Example](#file-input) | [📖](https://docs.praison.ai/docs/cli/file-input) |
| ↳ Final Agent | [Example](#final-agent) | [📖](https://docs.praison.ai/docs/cli/final-agent) |
| ↳ Max Tokens | [Example](#max-tokens) | [📖](https://docs.praison.ai/docs/cli/max-tokens) |
| **🧪 Evaluation** | | |
| ↳ Accuracy Evaluation | [Example](examples/eval/accuracy_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
| ↳ Performance Evaluation | [Example](examples/eval/performance_example.py) | [📖](https://docs.praison.ai/docs/cli/eval) |
+2 -2
View File
@@ -3,8 +3,8 @@ class Praisonai < Formula
desc "AI tools for various AI applications"
homepage "https://github.com/MervinPraison/PraisonAI"
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.76.tar.gz"
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.76.tar.gz | shasum -a 256`.split.first
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.81.tar.gz"
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.81.tar.gz | shasum -a 256`.split.first
license "MIT"
depends_on "[email protected]"
@@ -105,6 +105,7 @@ class FileSearchService:
- Fuzzy match file paths
- Cache results with TTL
- Respect .gitignore (basic)
- Support absolute paths (@/Users/...) and home paths (@~/...)
"""
def __init__(
@@ -125,9 +126,10 @@ class FileSearchService:
# Cache: query -> (results, timestamp)
self._cache: Dict[str, Tuple[List[FileSuggestion], float]] = {}
# File list cache
# File list cache per directory
self._file_list: Optional[List[str]] = None
self._file_list_time: float = 0
self._file_list_dir: Optional[str] = None
def _should_ignore(self, name: str) -> bool:
"""Check if file/dir should be ignored."""
@@ -136,20 +138,23 @@ class FileSearchService:
return True
return False
def _get_file_list(self) -> List[str]:
def _get_file_list(self, search_dir: str = None) -> List[str]:
"""Get list of all files (cached)."""
now = time.time()
target_dir = search_dir or self._root
# Return cached if fresh
if self._file_list is not None and (now - self._file_list_time) < self._cache_ttl:
# Return cached if fresh and same directory
if (self._file_list is not None and
self._file_list_dir == target_dir and
(now - self._file_list_time) < self._cache_ttl):
return self._file_list
files = []
try:
for root, dirs, filenames in os.walk(self._root):
for root, dirs, filenames in os.walk(target_dir):
# Calculate depth
rel_root = os.path.relpath(root, self._root)
rel_root = os.path.relpath(root, target_dir)
depth = 0 if rel_root == '.' else rel_root.count(os.sep) + 1
if depth > self._max_depth:
@@ -161,19 +166,20 @@ class FileSearchService:
# Add directories
for d in dirs:
rel_path = os.path.relpath(os.path.join(root, d), self._root)
rel_path = os.path.relpath(os.path.join(root, d), target_dir)
files.append(rel_path + os.sep)
# Add files
for f in filenames:
if not self._should_ignore(f):
rel_path = os.path.relpath(os.path.join(root, f), self._root)
rel_path = os.path.relpath(os.path.join(root, f), target_dir)
files.append(rel_path)
except OSError:
pass
self._file_list = files
self._file_list_time = now
self._file_list_dir = target_dir
return files
def _fuzzy_match(self, text: str, query: str) -> int:
@@ -221,6 +227,11 @@ class FileSearchService:
"""
Search for files matching query.
Supports:
- Relative paths: @file.txt, @src/main.py
- Home paths: @~/Documents, @~/.config
- Absolute paths: @/Users/name/file.txt, @/etc/hosts
Args:
query: Search query (fuzzy matched)
max_results: Maximum results to return
@@ -228,8 +239,51 @@ class FileSearchService:
Returns:
List of FileSuggestion sorted by score
"""
# Determine search directory based on query
search_dir = self._root
search_query = query
prefix = ""
# Handle home directory paths (@~/)
if query.startswith('~'):
expanded = os.path.expanduser(query)
# Find the directory part and filename part
if os.path.isdir(expanded):
search_dir = expanded
search_query = ""
prefix = query.rstrip('/') + '/'
else:
parent = os.path.dirname(expanded)
if os.path.isdir(parent):
search_dir = parent
search_query = os.path.basename(expanded)
prefix = os.path.dirname(query).rstrip('/') + '/'
else:
# Just expand ~ and search from home
search_dir = os.path.expanduser('~')
search_query = query[2:] if query.startswith('~/') else query[1:]
prefix = "~/"
# Handle absolute paths (@/Users/..., @/etc/...)
elif query.startswith('/'):
if os.path.isdir(query):
search_dir = query
search_query = ""
prefix = query.rstrip('/') + '/'
else:
parent = os.path.dirname(query)
if os.path.isdir(parent):
search_dir = parent
search_query = os.path.basename(query)
prefix = parent.rstrip('/') + '/'
else:
# Search from root with the query
search_dir = '/'
search_query = query[1:]
prefix = "/"
# Check cache
cache_key = f"{query}:{max_results}"
cache_key = f"{search_dir}:{search_query}:{max_results}"
now = time.time()
if cache_key in self._cache:
@@ -237,25 +291,26 @@ class FileSearchService:
if (now - cached_time) < self._cache_ttl:
return results
# Get all files
all_files = self._get_file_list()
# Get all files from the appropriate directory
all_files = self._get_file_list(search_dir)
# Score and filter
scored: List[Tuple[int, str]] = []
for path in all_files:
score = self._fuzzy_match(path, query)
score = self._fuzzy_match(path, search_query)
if score > 0:
scored.append((score, path))
# Sort by score descending
scored.sort(key=lambda x: (-x[0], x[1]))
# Build results
# Build results with proper prefix
results = []
for score, path in scored[:max_results]:
file_type = "directory" if path.endswith(os.sep) else "file"
display_path = prefix + path if prefix else path
results.append(FileSuggestion(
path=path,
path=display_path,
file_type=file_type,
score=score
))
+76 -10
View File
@@ -16,7 +16,7 @@ logger = logging.getLogger(__name__)
COMPARE_MODES = {
"basic": {},
"tools": {"tools": "internet_search"},
"tools": {"tools": "calculator,internet_search"},
"research": {"research": True},
"planning": {"planning": True},
"memory": {"memory": True},
@@ -177,12 +177,13 @@ def save_compare_result(result: CompareResult, path: str) -> bool:
return False
def format_comparison_table(result: CompareResult) -> str:
def format_comparison_table(result: CompareResult, show_responses: bool = True) -> str:
"""
Format comparison result as a table string.
Format comparison result as a table string with optional response display.
Args:
result: CompareResult to format
show_responses: Whether to show actual responses from each mode
Returns:
Formatted table string
@@ -190,10 +191,12 @@ def format_comparison_table(result: CompareResult) -> str:
try:
from rich.console import Console
from rich.table import Table
from rich.panel import Panel
from io import StringIO
console = Console(file=StringIO(), force_terminal=True)
# Main comparison table
table = Table(title=f"Comparison: {result.query[:50]}...")
table.add_column("Mode", style="cyan")
table.add_column("Time (ms)", style="green")
@@ -224,6 +227,26 @@ def format_comparison_table(result: CompareResult) -> str:
)
console.print(table)
# Show responses if enabled
if show_responses:
console.print("\n[bold cyan]📝 Responses:[/bold cyan]")
for c in result.comparisons:
if c.error is None:
# Truncate long responses
response_preview = c.output[:500] + "..." if len(c.output) > 500 else c.output
console.print(Panel(
response_preview,
title=f"[bold]{c.mode}[/bold] ({c.execution_time_ms:.0f}ms)",
border_style="cyan"
))
else:
console.print(Panel(
f"[red]Error: {c.error}[/red]",
title=f"[bold]{c.mode}[/bold]",
border_style="red"
))
return console.file.getvalue()
except ImportError:
lines = [f"Comparison: {result.query}"]
@@ -231,6 +254,8 @@ def format_comparison_table(result: CompareResult) -> str:
for c in result.comparisons:
status = "OK" if c.error is None else f"ERROR: {c.error}"
lines.append(f"{c.mode}: {c.execution_time_ms:.1f}ms - {status}")
if show_responses and c.error is None:
lines.append(f" Response: {c.output[:200]}...")
return "\n".join(lines)
@@ -407,13 +432,7 @@ class CompareHandler:
from praisonaiagents.tools import internet_search
tools.append(internet_search)
elif name == "calculator":
def calculator(expression: str) -> str:
"""Evaluate a mathematical expression."""
try:
return str(eval(expression))
except Exception as e:
return f"Error: {e}"
tools.append(calculator)
tools.append(_calculator_tool)
except ImportError:
logger.warning(f"Could not load tool: {name}")
@@ -466,3 +485,50 @@ class CompareHandler:
print(f"[red]Failed to save results to: {output_path}[/red]")
return result
def _calculator_tool(expression: str) -> str:
"""
Evaluate a mathematical expression safely.
Args:
expression: Mathematical expression to evaluate (e.g., "2+2", "15*3", "100/4")
Returns:
The result of the calculation as a string
"""
import ast
import operator
# Safe operators
operators = {
ast.Add: operator.add,
ast.Sub: operator.sub,
ast.Mult: operator.mul,
ast.Div: operator.truediv,
ast.Pow: operator.pow,
ast.USub: operator.neg,
ast.UAdd: operator.pos,
ast.Mod: operator.mod,
ast.FloorDiv: operator.floordiv,
}
def _eval(node):
if isinstance(node, ast.Constant):
return node.value
elif isinstance(node, ast.BinOp):
left = _eval(node.left)
right = _eval(node.right)
return operators[type(node.op)](left, right)
elif isinstance(node, ast.UnaryOp):
operand = _eval(node.operand)
return operators[type(node.op)](operand)
else:
raise ValueError(f"Unsupported operation: {type(node)}")
try:
tree = ast.parse(expression, mode='eval')
result = _eval(tree.body)
return str(result)
except Exception as e:
return f"Error evaluating '{expression}': {e}"
+109 -20
View File
@@ -26,42 +26,68 @@ class EvalHandler:
def run_accuracy(
self,
agent_file: str,
input_text: str,
expected_output: str,
agent_file: Optional[str] = None,
input_text: str = "",
expected_output: str = "",
iterations: int = 1,
model: Optional[str] = None,
output_file: Optional[str] = None
output_file: Optional[str] = None,
prompt: Optional[str] = None,
llm: Optional[str] = None
) -> Dict[str, Any]:
"""
Run accuracy evaluation on an agent.
Args:
agent_file: Path to agents.yaml file
agent_file: Path to agents.yaml file (optional if prompt is provided)
input_text: Input to provide to the agent
expected_output: Expected output to compare against
iterations: Number of evaluation iterations
model: LLM model for judging
output_file: Path to save results
prompt: Direct prompt (alternative to agent_file)
llm: LLM model for the agent (when using prompt)
Returns:
Evaluation result dictionary
"""
try:
from praisonaiagents.eval import AccuracyEvaluator
from praisonai.agents_generator import AgentsGenerator
from praisonaiagents import Agent
except ImportError as e:
logger.error(f"Failed to import evaluation modules: {e}")
return {"error": str(e)}
try:
generator = AgentsGenerator(agent_file)
agents = generator.generate_agents()
if not agents:
return {"error": "No agents found in configuration"}
agent = agents[0] if isinstance(agents, list) else agents
# Create agent either from file or from prompt
if prompt:
# Direct prompt mode - create agent on the fly
agent = Agent(
name="EvalAgent",
role="Assistant",
goal="Complete the given task",
backstory="You are a helpful assistant.",
llm=llm or model or "gpt-4o-mini",
verbose=False
)
# Use prompt as input if input_text not provided
if not input_text:
input_text = prompt
elif agent_file:
# Load from agents.yaml
try:
from praisonai.agents_generator import AgentsGenerator
generator = AgentsGenerator(agent_file)
agents = generator.generate_agents()
if not agents:
return {"error": "No agents found in configuration"}
agent = agents[0] if isinstance(agents, list) else agents
except Exception as e:
return {"error": f"Failed to load agents from {agent_file}: {e}"}
else:
return {"error": "Either --agent or --prompt must be provided"}
evaluator = AccuracyEvaluator(
agent=agent,
@@ -321,18 +347,49 @@ class EvalHandler:
return batch_result
def handle_eval_command(args) -> None:
def handle_eval_command(args) -> int:
"""
Handle the eval CLI command.
Args:
args: Parsed command line arguments
args: Command line arguments (list or parsed namespace)
Returns:
Exit code
"""
import argparse
# If args is a list, parse it first
if isinstance(args, list):
parser = argparse.ArgumentParser(prog="praisonai eval")
subparsers = parser.add_subparsers(dest='eval_type')
add_eval_parser_subcommands(subparsers)
try:
args = parser.parse_args(args)
except SystemExit:
return 1
if not args.eval_type:
parser.print_help()
print("\n[bold]Examples:[/bold]")
print(" praisonai eval accuracy --prompt \"What is 2+2?\" --expected \"4\"")
print(" praisonai eval performance --agent agents.yaml --input \"Hello\"")
return 0
handler = EvalHandler(verbose=getattr(args, 'verbose', False))
eval_type = getattr(args, 'eval_type', 'accuracy')
agent_file = getattr(args, 'agent', 'agents.yaml')
agent_file = getattr(args, 'agent', None)
output_file = getattr(args, 'output', None)
prompt = getattr(args, 'prompt', None)
llm = getattr(args, 'llm', None)
# If no agent file and no prompt, check if agents.yaml exists
if not agent_file and not prompt:
import os
if os.path.exists('agents.yaml'):
agent_file = 'agents.yaml'
if eval_type == 'accuracy':
result = handler.run_accuracy(
@@ -341,7 +398,9 @@ def handle_eval_command(args) -> None:
expected_output=getattr(args, 'expected', ''),
iterations=getattr(args, 'iterations', 1),
model=getattr(args, 'model', None),
output_file=output_file
output_file=output_file,
prompt=prompt,
llm=llm
)
elif eval_type == 'performance':
result = handler.run_performance(
@@ -384,12 +443,40 @@ def handle_eval_command(args) -> None:
)
else:
print(f"Unknown evaluation type: {eval_type}")
return
return 1
if 'error' in result:
print(f"Error: {result['error']}")
return 1
elif not getattr(args, 'quiet', False):
print(json.dumps(result, indent=2))
return 0
def add_eval_parser_subcommands(subparsers) -> None:
"""Add eval subcommand parsers to an existing subparsers object."""
accuracy_parser = subparsers.add_parser('accuracy', help='Run accuracy evaluation')
accuracy_parser.add_argument('--agent', '-a', help='Agent config file (optional if --prompt used)')
accuracy_parser.add_argument('--prompt', '-p', type=str, help='Direct prompt (alternative to --agent)')
accuracy_parser.add_argument('--llm', help='LLM model for agent (when using --prompt)')
accuracy_parser.add_argument('--input', '-i', help='Input text (defaults to --prompt if not provided)')
accuracy_parser.add_argument('--expected', '-e', required=True, help='Expected output')
accuracy_parser.add_argument('--iterations', '-n', type=int, default=1, help='Number of iterations')
accuracy_parser.add_argument('--model', '-m', help='Judge model')
accuracy_parser.add_argument('--output', '-o', help='Output file')
accuracy_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
accuracy_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
perf_parser = subparsers.add_parser('performance', help='Run performance evaluation')
perf_parser.add_argument('--agent', '-a', default='agents.yaml', help='Agent config file')
perf_parser.add_argument('--input', '-i', default='Hello', help='Input text')
perf_parser.add_argument('--iterations', '-n', type=int, default=10, help='Number of iterations')
perf_parser.add_argument('--warmup', '-w', type=int, default=2, help='Warmup runs')
perf_parser.add_argument('--memory', action='store_true', default=True, help='Track memory')
perf_parser.add_argument('--output', '-o', help='Output file')
perf_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
perf_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
def add_eval_parser(subparsers) -> None:
@@ -410,8 +497,10 @@ def add_eval_parser(subparsers) -> None:
'accuracy',
help='Run accuracy evaluation'
)
accuracy_parser.add_argument('--agent', '-a', default='agents.yaml', help='Agent config file')
accuracy_parser.add_argument('--input', '-i', required=True, help='Input text')
accuracy_parser.add_argument('--agent', '-a', help='Agent config file (optional if --prompt used)')
accuracy_parser.add_argument('--prompt', '-p', help='Direct prompt (alternative to --agent)')
accuracy_parser.add_argument('--llm', help='LLM model for agent (when using --prompt)')
accuracy_parser.add_argument('--input', '-i', help='Input text (defaults to --prompt if not provided)')
accuracy_parser.add_argument('--expected', '-e', required=True, help='Expected output')
accuracy_parser.add_argument('--iterations', '-n', type=int, default=1, help='Number of iterations')
accuracy_parser.add_argument('--model', '-m', help='Judge model')
@@ -3,12 +3,21 @@ Fast Context Handler for CLI.
Provides codebase search capability using FastContext.
Usage: praisonai "Find authentication code" --fast-context ./src
Fast Context Strategy:
1. Extract search keywords from natural language query using LLM
2. Provide folder tree context to help understand codebase structure
3. Use both file name search AND content search (ripgrep-style)
4. Return relevant code snippets for context
"""
import os
import logging
from typing import Any, Dict, Tuple, List
from .base import FlagHandler
logger = logging.getLogger(__name__)
class FastContextHandler(FlagHandler):
"""
@@ -61,14 +70,208 @@ class FastContextHandler(FlagHandler):
return True, ""
def _count_files_quick(self, path: str, max_count: int = 1000) -> int:
"""Quickly count files in directory, stopping at max_count."""
count = 0
ignore_dirs = {'.git', '__pycache__', 'node_modules', '.venv', 'venv', '.cache', 'dist', 'build'}
for root, dirs, files in os.walk(path):
dirs[:] = [d for d in dirs if d not in ignore_dirs and not d.startswith('.')]
count += len(files)
if count >= max_count:
return count
return count
def _get_folder_tree(self, path: str, max_depth: int = 2) -> str:
"""
Get a compact folder tree structure for context.
Optimized for large directories:
- Limits depth to max_depth (default 2)
- Shows max 10 files per directory
- Truncates total output to 50 lines
- Skips common non-code directories
"""
# Quick check for very large directories
file_count = self._count_files_quick(path, 500)
if file_count >= 500:
# For large directories, reduce depth and show summary
max_depth = 1
logger.debug(f"Large directory detected ({file_count}+ files), reducing tree depth")
tree_lines = []
ignore_dirs = {'.git', '__pycache__', 'node_modules', '.venv', 'venv', '.cache', 'dist', 'build', 'coverage', '.pytest_cache', '.mypy_cache', 'eggs', '*.egg-info'}
ignore_exts = {'.pyc', '.pyo', '.so', '.o', '.a', '.dylib', '.class', '.jar'}
total_files = 0
max_total_files = 100 # Stop after seeing this many files
def walk_tree(current_path, prefix="", depth=0):
nonlocal total_files
if depth > max_depth or total_files >= max_total_files:
return
try:
entries = sorted(os.listdir(current_path))
except PermissionError:
return
dirs = []
files = []
for entry in entries:
full_path = os.path.join(current_path, entry)
if os.path.isdir(full_path):
if entry not in ignore_dirs and not entry.startswith('.'):
dirs.append(entry)
else:
ext = os.path.splitext(entry)[1]
if ext not in ignore_exts and not entry.startswith('.'):
files.append(entry)
# Limit directories shown at each level
dirs_to_show = dirs[:15]
for i, d in enumerate(dirs_to_show):
is_last = (i == len(dirs_to_show) - 1) and not files
tree_lines.append(f"{prefix}{'└── ' if is_last else '├── '}{d}/")
walk_tree(os.path.join(current_path, d),
prefix + (' ' if is_last else '│ '), depth + 1)
if len(dirs) > 15:
tree_lines.append(f"{prefix} ... and {len(dirs) - 15} more directories")
# Limit files shown
files_to_show = files[:10]
total_files += len(files)
for i, f in enumerate(files_to_show):
is_last = i == len(files_to_show) - 1
tree_lines.append(f"{prefix}{'└── ' if is_last else '├── '}{f}")
if len(files) > 10:
tree_lines.append(f"{prefix} ... and {len(files) - 10} more files")
tree_lines.append(os.path.basename(path) + "/")
walk_tree(path)
return "\n".join(tree_lines[:50]) # Limit total lines
def _extract_keywords(self, query: str, folder_tree: str) -> List[str]:
"""Extract search keywords from natural language query using LLM."""
try:
from praisonaiagents import Agent
extractor = Agent(
name="KeywordExtractor",
role="Search Query Analyzer",
goal="Extract file search keywords from natural language",
backstory="Expert at understanding code search queries",
llm="gpt-4o-mini",
verbose=False
)
prompt = f"""Given this folder structure:
{folder_tree}
And this search query: "{query}"
Extract 1-5 specific keywords/patterns to search for in file names and content.
Return ONLY the keywords, one per line, no explanations.
Focus on: file names, function names, class names, variable names that might match.
If the query mentions a folder name that exists, include files from that folder."""
response = extractor.chat(prompt, stream=False)
keywords = [k.strip() for k in str(response).strip().split('\n') if k.strip()]
logger.debug(f"Extracted keywords: {keywords}")
return keywords[:5] # Limit to 5 keywords
except Exception as e:
logger.debug(f"Keyword extraction failed: {e}")
# Fallback: extract words from query
return [w for w in query.split() if len(w) > 2][:3]
def _search_files_by_name(self, path: str, keywords: List[str]) -> List[Dict[str, Any]]:
"""Search for files matching keywords in their names."""
matches = []
path = os.path.abspath(path)
ignore_dirs = {'.git', '__pycache__', 'node_modules', '.venv', 'venv'}
for root, dirs, files in os.walk(path):
dirs[:] = [d for d in dirs if d not in ignore_dirs]
for filename in files:
filepath = os.path.join(root, filename)
rel_path = os.path.relpath(filepath, path)
# Check if any keyword matches filename or path
for kw in keywords:
kw_lower = kw.lower()
if kw_lower in filename.lower() or kw_lower in rel_path.lower():
matches.append({
'file': rel_path,
'lines': [(1, 50)], # First 50 lines
'relevance': 1.0,
'match_type': 'filename'
})
break
return matches[:20] # Limit results
def _search_files_by_content(self, path: str, keywords: List[str]) -> List[Dict[str, Any]]:
"""Search for files containing keywords in content."""
matches = []
path = os.path.abspath(path)
ignore_dirs = {'.git', '__pycache__', 'node_modules', '.venv', 'venv'}
code_exts = {'.py', '.js', '.ts', '.jsx', '.tsx', '.java', '.go', '.rs', '.rb', '.sh', '.yaml', '.yml', '.json', '.md', '.txt'}
for root, dirs, files in os.walk(path):
dirs[:] = [d for d in dirs if d not in ignore_dirs]
for filename in files:
ext = os.path.splitext(filename)[1]
if ext not in code_exts:
continue
filepath = os.path.join(root, filename)
rel_path = os.path.relpath(filepath, path)
try:
with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
content = f.read(50000) # Read first 50KB
content_lower = content.lower()
for kw in keywords:
if kw.lower() in content_lower:
# Find line numbers with matches
lines = content.split('\n')
match_lines = []
for i, line in enumerate(lines[:200], 1): # Check first 200 lines
if kw.lower() in line.lower():
match_lines.append((max(1, i-2), min(len(lines), i+2)))
if match_lines:
matches.append({
'file': rel_path,
'lines': match_lines[:5], # Limit line ranges
'relevance': 0.8,
'match_type': 'content'
})
break
except Exception:
continue
return matches[:15] # Limit results
def search_context(self, query: str, path: str, **kwargs) -> List[Dict[str, Any]]:
"""
Search for relevant context in the codebase.
Uses a smart multi-step approach:
1. Get folder tree structure
2. Use LLM to extract search keywords from query
3. Search by filename AND content
4. Return combined results
Args:
query: Search query
query: Natural language search query
path: Path to search
**kwargs: Additional search options
- use_llm_keywords: Use LLM to extract keywords (default: True)
Returns:
List of matching results
@@ -84,32 +287,50 @@ class FastContextHandler(FlagHandler):
return []
try:
from praisonaiagents import FastContext
use_llm_keywords = kwargs.get('use_llm_keywords', True)
fc = FastContext(
path=path,
model=kwargs.get('model', 'gpt-4o-mini'),
max_turns=kwargs.get('max_turns', 4),
parallelism=kwargs.get('parallelism', 8),
timeout=kwargs.get('timeout', 30.0)
)
# Step 1: Get folder tree for context
logger.debug(f"Getting folder tree for: {path}")
folder_tree = self._get_folder_tree(path)
logger.debug(f"Folder tree:\n{folder_tree}")
result = fc.search(query)
# Step 2: Extract keywords
if use_llm_keywords:
logger.debug(f"Extracting keywords from query: {query}")
keywords = self._extract_keywords(query, folder_tree)
else:
keywords = [w for w in query.split() if len(w) > 2][:3]
# Convert to list of dicts
logger.debug(f"Search keywords: {keywords}")
if not keywords:
self.print_status("Could not extract search keywords", "warning")
return []
# Step 3: Search by filename
logger.debug("Searching by filename...")
filename_matches = self._search_files_by_name(path, keywords)
logger.debug(f"Found {len(filename_matches)} filename matches")
# Step 4: Search by content
logger.debug("Searching by content...")
content_matches = self._search_files_by_content(path, keywords)
logger.debug(f"Found {len(content_matches)} content matches")
# Combine and deduplicate
seen_files = set()
matches = []
if hasattr(result, 'files'):
for file_match in result.files:
matches.append({
'file': file_match.path,
'lines': [(lr.start, lr.end) for lr in file_match.line_ranges],
'relevance': getattr(file_match, 'relevance', 1.0)
})
for m in filename_matches + content_matches:
if m['file'] not in seen_files:
seen_files.add(m['file'])
matches.append(m)
self.print_status(f"🔍 Found {len(matches)} relevant files", "success")
return matches
except Exception as e:
logger.error(f"FastContext search error: {e}")
self.log(f"FastContext search error: {e}", "error")
return []
@@ -75,7 +75,7 @@ class GuardrailHandler(FlagHandler):
Args:
description: Natural language description of validation criteria
llm: Optional LLM instance for validation
llm: Optional LLM instance for validation (defaults to gpt-4o-mini)
Returns:
LLMGuardrail instance or None if unavailable
@@ -87,6 +87,12 @@ class GuardrailHandler(FlagHandler):
)
return None
# Provide default LLM if none specified
if llm is None:
import os
# Try to get from environment or use default
llm = os.environ.get('OPENAI_MODEL_NAME', 'gpt-4o-mini')
return LLMGuardrail(description=description, llm=llm)
def validate_output(self, output: str, description: str, llm: Any = None) -> Tuple[bool, str]:
+35 -16
View File
@@ -72,11 +72,11 @@ class HandoffHandler(FlagHandler):
self.print_status(msg, "error")
return []
from praisonaiagents import Agent, Handoff
from praisonaiagents import Agent, handoff
agents = []
# Create agents for each role
# First pass: Create all agents without handoffs
for i, name in enumerate(agent_names):
agent_config = {
"name": name.title().replace("_", " "),
@@ -91,26 +91,45 @@ class HandoffHandler(FlagHandler):
agent = Agent(**agent_config)
agents.append(agent)
# Set up handoffs between agents
for i, agent in enumerate(agents[:-1]):
next_agent = agents[i + 1]
handoff = Handoff(
target=next_agent,
name=f"handoff_to_{agent_names[i + 1]}",
description=f"Hand off task to {agent_names[i + 1]} for further processing"
)
# Second pass: Set up handoffs by recreating agents with handoff config
final_agents = []
for i, name in enumerate(agent_names):
agent_config = {
"name": name.title().replace("_", " "),
"role": name.title().replace("_", " "),
"goal": f"Complete tasks as {name}",
"backstory": f"Expert {name} with specialized skills"
}
# Add handoff to agent
if not hasattr(agent, 'handoffs') or agent.handoffs is None:
agent.handoffs = []
agent.handoffs.append(handoff)
if llm:
agent_config["llm"] = llm
# Add handoff to next agent if not the last agent
if i < len(agent_names) - 1:
next_agent = agents[i + 1]
next_agent_name = agent_names[i + 1]
# Pass handoffs via constructor - this properly registers them as tools
agent_config["handoffs"] = [
handoff(
agent=next_agent,
tool_name_override=f"handoff_to_{next_agent_name}",
tool_description_override=f"Hand off the task to {next_agent_name}. Call this after completing your part."
)
]
# Add instructions to use handoff
agent_config["instructions"] = f"After completing your task, you MUST use the handoff_to_{next_agent_name} tool to pass your work to the next agent."
final_agent = Agent(**agent_config)
final_agents.append(final_agent)
self.print_status(f"🤝 Created {len(agents)} agents with handoff chain", "success")
self.print_status(f"🤝 Created {len(final_agents)} agents with handoff chain", "success")
for i, name in enumerate(agent_names):
arrow = " → " if i < len(agent_names) - 1 else ""
self.print_status(f" {name}{arrow}", "info")
return agents
return final_agents
def apply_to_agent_config(self, config: Dict[str, Any], flag_value: Any) -> Dict[str, Any]:
"""
+266 -89
View File
@@ -1,32 +1,52 @@
"""
Image Handler for CLI.
Provides image processing capabilities using ImageAgent.
Usage: praisonai "Describe this image" --image path/to/image.png
Provides two distinct image capabilities:
1. Image Description (Vision) - Analyze existing images using vision-capable LLMs
Usage: praisonai "Describe this image" --image path/to/image.png
2. Image Generation - Generate new images from text prompts
Usage: praisonai "A sunset over mountains" --image-generate
"""
import os
from typing import Any, Dict, Tuple, List
import base64
import logging
from typing import Any, Dict, Tuple, List, Optional
from .base import FlagHandler
# Supported image extensions
SUPPORTED_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.svg'}
logger = logging.getLogger(__name__)
# Supported image extensions for vision/description
SUPPORTED_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.svg', '.tiff'}
# Default vision-capable models (gpt-4o has better vision support than gpt-4o-mini)
DEFAULT_VISION_MODEL = "gpt-4o"
# Default image generation models
DEFAULT_IMAGE_GEN_MODEL = "dall-e-3"
SUPPORTED_IMAGE_GEN_MODELS = [
'gpt-image-1', 'gpt-image-1-mini', 'gpt-image-1.5',
'chatgpt-image-latest', 'dall-e-2', 'dall-e-3'
]
class ImageHandler(FlagHandler):
"""
Handler for --image flag.
Handler for --image flag (Image Description/Vision).
Processes images using ImageAgent for vision-based tasks.
Describes/analyzes existing images using vision-capable LLMs like GPT-4o.
This is for understanding what's IN an image, not generating new images.
Example:
praisonai "Describe this image" --image photo.png
praisonai "What's in this image?" --image /path/to/image.jpg
praisonai "Count the objects" --image image1.png,image2.png
"""
@property
def feature_name(self) -> str:
return "image"
return "image_describe"
@property
def flag_name(self) -> str:
@@ -34,17 +54,15 @@ class ImageHandler(FlagHandler):
@property
def flag_help(self) -> str:
return "Path to image file for vision-based tasks"
return "Path to image file for vision-based description/analysis"
def check_dependencies(self) -> Tuple[bool, str]:
"""Check if ImageAgent is available."""
"""Check if litellm is available for vision."""
try:
import importlib.util
if importlib.util.find_spec("praisonaiagents") is not None:
return True, ""
return False, "praisonaiagents not installed. Install with: pip install praisonaiagents"
import litellm
return True, ""
except ImportError:
return False, "praisonaiagents not installed. Install with: pip install praisonaiagents"
return False, "litellm not installed. Install with: pip install litellm"
def validate_image_path(self, path: str) -> Tuple[bool, str]:
"""
@@ -71,127 +89,154 @@ class ImageHandler(FlagHandler):
return True, ""
def _encode_image_to_base64(self, image_path: str) -> str:
"""Encode a local image file to base64."""
with open(image_path, 'rb') as f:
return base64.b64encode(f.read()).decode('utf-8')
def _get_image_mime_type(self, image_path: str) -> str:
"""Get MIME type based on file extension."""
ext = os.path.splitext(image_path)[1].lower()
mime_types = {
'.png': 'image/png',
'.jpg': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.gif': 'image/gif',
'.webp': 'image/webp',
'.bmp': 'image/bmp',
'.svg': 'image/svg+xml',
'.tiff': 'image/tiff',
}
return mime_types.get(ext, 'image/png')
def _build_image_content(self, image_path: str) -> Dict[str, Any]:
"""Build image content for vision API."""
if image_path.startswith(('http://', 'https://')):
return {
"type": "image_url",
"image_url": {"url": image_path}
}
else:
mime_type = self._get_image_mime_type(image_path)
base64_data = self._encode_image_to_base64(image_path)
return {
"type": "image_url",
"image_url": {"url": f"data:{mime_type};base64,{base64_data}"}
}
def apply_to_agent_config(self, config: Dict[str, Any], flag_value: Any) -> Dict[str, Any]:
"""
Apply image configuration to agent config.
Args:
config: Agent configuration dictionary
flag_value: Image path or list of paths
Returns:
Modified configuration with image settings
"""
"""Apply image configuration to agent config."""
if flag_value:
# Store image path(s) for use with ImageAgent
if isinstance(flag_value, str):
config["images"] = [flag_value]
elif isinstance(flag_value, list):
config["images"] = flag_value
config["use_image_agent"] = True
config["use_vision"] = True
return config
def create_image_agent(self, llm: str = None, **kwargs) -> Any:
def describe_image(self, prompt: str, image_path: str, llm = None) -> str:
"""
Create an ImageAgent instance.
Args:
llm: Optional LLM model name
**kwargs: Additional agent configuration
Returns:
ImageAgent instance or None if unavailable
"""
available, msg = self.check_dependencies()
if not available:
self.print_status(msg, "error")
return None
from praisonaiagents import ImageAgent
agent_config = {
"name": "ImageAnalyzer",
"role": "Image Analyst",
"goal": "Analyze and describe images accurately",
"backstory": "Expert at understanding visual content"
}
if llm:
agent_config["llm"] = llm
agent_config.update(kwargs)
return ImageAgent(**agent_config)
def process_image(self, prompt: str, image_path: str, llm: str = None) -> str:
"""
Process an image with the given prompt.
Describe an image using vision-capable LLM.
Args:
prompt: The prompt/question about the image
image_path: Path to the image file
llm: Optional LLM model name
llm: Optional LLM model name (must be vision-capable), can be string or dict
Returns:
Agent's response about the image
Description of the image
"""
# Validate image path
valid, msg = self.validate_image_path(image_path)
if not valid:
return f"Error: {msg}"
agent = self.create_image_agent(llm=llm)
if not agent:
return "Error: Could not create ImageAgent"
available, dep_msg = self.check_dependencies()
if not available:
return f"Error: {dep_msg}"
import litellm
# Handle llm being a dict (from agent_config) or string
if isinstance(llm, dict):
model = llm.get('model', DEFAULT_VISION_MODEL)
else:
model = llm or DEFAULT_VISION_MODEL
try:
# ImageAgent expects images as a list
result = agent.chat(prompt, images=[image_path])
return result
# Build message with image
content = [
{"type": "text", "text": prompt},
self._build_image_content(image_path)
]
response = litellm.completion(
model=model,
messages=[{"role": "user", "content": content}]
)
return response.choices[0].message.content
except Exception as e:
self.log(f"Image processing error: {e}", "error")
return f"Error processing image: {e}"
logger.error(f"Error describing image: {e}")
return f"Error describing image: {e}"
def process_multiple_images(self, prompt: str, image_paths: List[str], llm: str = None) -> str:
def describe_multiple_images(self, prompt: str, image_paths: List[str], llm = None) -> str:
"""
Process multiple images with the given prompt.
Describe multiple images using vision-capable LLM.
Args:
prompt: The prompt/question about the images
image_paths: List of paths to image files
llm: Optional LLM model name
llm: Optional LLM model name, can be string or dict
Returns:
Agent's response about the images
Description of the images
"""
# Validate all image paths
for path in image_paths:
valid, msg = self.validate_image_path(path)
if not valid:
return f"Error: {msg}"
agent = self.create_image_agent(llm=llm)
if not agent:
return "Error: Could not create ImageAgent"
available, dep_msg = self.check_dependencies()
if not available:
return f"Error: {dep_msg}"
import litellm
# Handle llm being a dict (from agent_config) or string
if isinstance(llm, dict):
model = llm.get('model', DEFAULT_VISION_MODEL)
else:
model = llm or DEFAULT_VISION_MODEL
try:
result = agent.chat(prompt, images=image_paths)
return result
# Build message with all images
content = [{"type": "text", "text": prompt}]
for path in image_paths:
content.append(self._build_image_content(path))
response = litellm.completion(
model=model,
messages=[{"role": "user", "content": content}]
)
return response.choices[0].message.content
except Exception as e:
self.log(f"Image processing error: {e}", "error")
return f"Error processing images: {e}"
logger.error(f"Error describing images: {e}")
return f"Error describing images: {e}"
def execute(self, prompt: str = None, image_path: str = None, llm: str = None, **kwargs) -> str:
"""
Execute image processing.
Execute image description (vision).
Args:
prompt: The prompt/question about the image
image_path: Path to the image file (or comma-separated paths)
llm: Optional LLM model name
llm: Optional vision-capable LLM model name
Returns:
Agent's response
Description of the image(s)
"""
if not prompt:
return "Error: No prompt provided"
@@ -202,6 +247,138 @@ class ImageHandler(FlagHandler):
# Handle multiple images (comma-separated)
if ',' in image_path:
paths = [p.strip() for p in image_path.split(',')]
return self.process_multiple_images(prompt, paths, llm)
return self.describe_multiple_images(prompt, paths, llm)
return self.process_image(prompt, image_path, llm)
return self.describe_image(prompt, image_path, llm)
class ImageGenerateHandler(FlagHandler):
"""
Handler for --image-generate flag (Image Generation).
Generates NEW images from text prompts using DALL-E or similar models.
This is for CREATING images, not analyzing existing ones.
Example:
praisonai "A sunset over mountains" --image-generate
praisonai "A cat wearing a hat" --image-generate --llm dall-e-3
"""
@property
def feature_name(self) -> str:
return "image_generate"
@property
def flag_name(self) -> str:
return "image_generate"
@property
def flag_help(self) -> str:
return "Generate an image from the text prompt"
def check_dependencies(self) -> Tuple[bool, str]:
"""Check if ImageAgent is available."""
try:
import importlib.util
if importlib.util.find_spec("praisonaiagents") is not None:
return True, ""
return False, "praisonaiagents not installed. Install with: pip install praisonaiagents"
except ImportError:
return False, "praisonaiagents not installed. Install with: pip install praisonaiagents"
def apply_to_agent_config(self, config: Dict[str, Any], flag_value: Any) -> Dict[str, Any]:
"""Apply image generation configuration."""
if flag_value:
config["use_image_generation"] = True
return config
def generate_image(self, prompt: str, llm: str = None, output_path: str = None) -> Dict[str, Any]:
"""
Generate an image from a text prompt.
Args:
prompt: Text description of the image to generate
llm: Image generation model (dall-e-3, dall-e-2, etc.)
output_path: Optional path to save the generated image
Returns:
Dict with image URL or path, or error message
"""
available, msg = self.check_dependencies()
if not available:
return {"error": msg}
from praisonaiagents import ImageAgent
# Use provided model or default
model = llm or DEFAULT_IMAGE_GEN_MODEL
# Validate model is supported for generation
if model not in SUPPORTED_IMAGE_GEN_MODELS:
return {
"error": f"Invalid model: '{model}'. Supported image generation models: {', '.join(SUPPORTED_IMAGE_GEN_MODELS)}"
}
try:
agent = ImageAgent(
name="ImageGenerator",
role="Image Generation Assistant",
goal="Generate high-quality images from text descriptions",
llm=model
)
result = agent.chat(prompt)
# Save image if output path provided
if output_path and isinstance(result, dict) and 'data' in result:
self._save_image(result, output_path)
result['saved_to'] = output_path
return result
except Exception as e:
logger.error(f"Error generating image: {e}")
return {"error": str(e)}
def _save_image(self, result: Dict[str, Any], output_path: str) -> bool:
"""Save generated image to file."""
try:
import requests
if 'data' in result and len(result['data']) > 0:
image_data = result['data'][0]
if 'url' in image_data:
# Download from URL
response = requests.get(image_data['url'])
with open(output_path, 'wb') as f:
f.write(response.content)
return True
elif 'b64_json' in image_data:
# Decode base64
import base64
img_bytes = base64.b64decode(image_data['b64_json'])
with open(output_path, 'wb') as f:
f.write(img_bytes)
return True
return False
except Exception as e:
logger.error(f"Error saving image: {e}")
return False
def execute(self, prompt: str = None, llm: str = None, output_path: str = None, **kwargs) -> Dict[str, Any]:
"""
Execute image generation.
Args:
prompt: Text description of the image to generate
llm: Image generation model name
output_path: Optional path to save the image
Returns:
Dict with generated image info or error
"""
if not prompt:
return {"error": "No prompt provided for image generation"}
return self.generate_image(prompt, llm, output_path)
@@ -107,6 +107,14 @@ class MetricsHandler(FlagHandler):
if hasattr(agent, 'last_token_metrics'):
metrics.update(agent.last_token_metrics or {})
# Check llm_model property for model info
if hasattr(agent, 'llm_model'):
llm_model = agent.llm_model
if hasattr(llm_model, 'model'):
metrics['model'] = llm_model.model
elif isinstance(llm_model, str):
metrics['model'] = llm_model
if hasattr(agent, 'llm') and agent.llm:
llm = agent.llm
if hasattr(llm, 'last_token_metrics'):
@@ -117,6 +125,42 @@ class MetricsHandler(FlagHandler):
if hasattr(agent, 'metrics') and isinstance(agent.metrics, dict):
metrics.update(agent.metrics or {})
# Try to get from litellm's success callback data
try:
import litellm
# Check if there's accumulated usage data
if hasattr(litellm, '_thread_context') and litellm._thread_context:
ctx = litellm._thread_context
if hasattr(ctx, 'usage'):
usage = ctx.usage
if hasattr(usage, 'prompt_tokens'):
metrics['prompt_tokens'] = usage.prompt_tokens
if hasattr(usage, 'completion_tokens'):
metrics['completion_tokens'] = usage.completion_tokens
if hasattr(usage, 'total_tokens'):
metrics['total_tokens'] = usage.total_tokens
except:
pass
# Try to get cost from litellm cost tracking
try:
import litellm
from litellm import completion_cost
# If we have token counts, estimate cost
if 'prompt_tokens' in metrics and 'completion_tokens' in metrics:
model = metrics.get('model', 'gpt-4o-mini')
try:
cost = completion_cost(
model=model,
prompt_tokens=metrics['prompt_tokens'],
completion_tokens=metrics['completion_tokens']
)
metrics['cost'] = cost
except:
pass
except:
pass
return metrics
def post_process_result(self, result: Any, flag_value: Any) -> Any:
@@ -0,0 +1,226 @@
"""
Profile Handler for CLI.
Provides performance profiling and benchmarking capabilities.
Usage: praisonai profile <subcommand> [options]
"""
import argparse
import time
import json
from typing import Any, Dict, List, Optional
from dataclasses import dataclass, field
from datetime import datetime
@dataclass
class ProfileResult:
"""Result from a profiling run."""
prompt: str
response: str
execution_time_ms: float
model: str
tokens: Optional[Dict[str, int]] = None
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
def to_dict(self) -> Dict[str, Any]:
return {
"prompt": self.prompt,
"response": self.response[:200] + "..." if len(self.response) > 200 else self.response,
"execution_time_ms": self.execution_time_ms,
"model": self.model,
"tokens": self.tokens,
"timestamp": self.timestamp
}
def run_benchmark(prompt: str, model: str = None, iterations: int = 1, verbose: bool = False) -> List[ProfileResult]:
"""
Run a benchmark on a prompt.
Args:
prompt: The prompt to benchmark
model: Optional model to use
iterations: Number of iterations to run
verbose: Enable verbose output
Returns:
List of ProfileResult objects
"""
try:
from praisonaiagents import Agent
except ImportError:
print("[red]ERROR: praisonaiagents not installed. Install with: pip install praisonaiagents[/red]")
return []
results = []
agent_config = {
"name": "BenchmarkAgent",
"role": "Benchmark Assistant",
"goal": "Complete tasks efficiently",
"verbose": verbose
}
if model:
agent_config["llm"] = model
agent = Agent(**agent_config)
for i in range(iterations):
if verbose:
print(f"[dim]Running iteration {i + 1}/{iterations}...[/dim]")
start_time = time.time()
response = agent.start(prompt)
end_time = time.time()
execution_time_ms = (end_time - start_time) * 1000
result = ProfileResult(
prompt=prompt,
response=str(response),
execution_time_ms=execution_time_ms,
model=model or "default"
)
results.append(result)
return results
def display_benchmark_results(results: List[ProfileResult], output_format: str = "table"):
"""Display benchmark results."""
if not results:
print("[yellow]No results to display[/yellow]")
return
try:
from rich.console import Console
from rich.table import Table
console = Console()
except ImportError:
# Fallback to simple output
for r in results:
print(f"Time: {r.execution_time_ms:.1f}ms | Model: {r.model}")
return
if output_format == "json":
print(json.dumps([r.to_dict() for r in results], indent=2))
return
# Calculate statistics
times = [r.execution_time_ms for r in results]
avg_time = sum(times) / len(times)
min_time = min(times)
max_time = max(times)
table = Table(title="📊 Benchmark Results")
table.add_column("Metric", style="cyan")
table.add_column("Value", style="green")
table.add_row("Iterations", str(len(results)))
table.add_row("Average Time", f"{avg_time:.1f}ms")
table.add_row("Min Time", f"{min_time:.1f}ms")
table.add_row("Max Time", f"{max_time:.1f}ms")
table.add_row("Model", results[0].model)
console.print(table)
# Show response preview
if results:
console.print("\n[bold]Response Preview:[/bold]")
preview = results[0].response[:300] + "..." if len(results[0].response) > 300 else results[0].response
console.print(f"[dim]{preview}[/dim]")
def handle_profile_command(args: List[str]) -> int:
"""
Handle the profile command.
Args:
args: Command line arguments
Returns:
Exit code
"""
parser = argparse.ArgumentParser(prog="praisonai profile", description="Performance profiling and benchmarking")
subparsers = parser.add_subparsers(dest='subcommand', help='Profile subcommands')
# Benchmark subcommand
bench_parser = subparsers.add_parser('benchmark', aliases=['bench', 'run'], help='Run a benchmark')
bench_parser.add_argument('prompt', nargs='?', help='Prompt to benchmark')
bench_parser.add_argument('--model', '-m', type=str, help='Model to use')
bench_parser.add_argument('--iterations', '-n', type=int, default=1, help='Number of iterations')
bench_parser.add_argument('--output', '-o', type=str, help='Output file path')
bench_parser.add_argument('--format', '-f', type=str, choices=['table', 'json', 'html'], default='table', help='Output format')
bench_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
# Compare subcommand
compare_parser = subparsers.add_parser('compare', help='Compare models')
compare_parser.add_argument('prompt', nargs='?', help='Prompt to compare')
compare_parser.add_argument('--models', type=str, help='Comma-separated list of models to compare')
# Report subcommand
report_parser = subparsers.add_parser('report', help='Generate a profiling report')
report_parser.add_argument('--input', '-i', type=str, help='Input JSON file with results')
report_parser.add_argument('--output', '-o', type=str, help='Output file path')
report_parser.add_argument('--format', '-f', type=str, choices=['html', 'json', 'markdown'], default='html', help='Report format')
try:
parsed_args = parser.parse_args(args)
except SystemExit:
return 1
if not parsed_args.subcommand:
parser.print_help()
print("\n[bold]Examples:[/bold]")
print(" praisonai profile benchmark \"What is 2+2?\"")
print(" praisonai profile benchmark \"Write a poem\" --model gpt-4o --iterations 3")
print(" praisonai profile compare \"Explain AI\" --models gpt-4o,gpt-4o-mini")
return 0
if parsed_args.subcommand in ['benchmark', 'bench', 'run']:
if not parsed_args.prompt:
print("[red]ERROR: Please provide a prompt to benchmark[/red]")
print("Usage: praisonai profile benchmark \"Your prompt here\"")
return 1
print(f"[bold cyan]🔬 Running benchmark...[/bold cyan]")
results = run_benchmark(
parsed_args.prompt,
model=parsed_args.model,
iterations=parsed_args.iterations,
verbose=parsed_args.verbose
)
display_benchmark_results(results, parsed_args.format)
if parsed_args.output:
with open(parsed_args.output, 'w') as f:
json.dump([r.to_dict() for r in results], f, indent=2)
print(f"[green]Results saved to {parsed_args.output}[/green]")
return 0
elif parsed_args.subcommand == 'compare':
if not parsed_args.prompt:
print("[red]ERROR: Please provide a prompt to compare[/red]")
return 1
models = parsed_args.models.split(',') if parsed_args.models else ['gpt-4o-mini', 'gpt-4o']
print(f"[bold cyan]🔬 Comparing models: {', '.join(models)}[/bold cyan]")
all_results = []
for model in models:
print(f"[dim]Testing {model}...[/dim]")
results = run_benchmark(parsed_args.prompt, model=model, iterations=1)
all_results.extend(results)
display_benchmark_results(all_results)
return 0
elif parsed_args.subcommand == 'report':
print("[yellow]Report generation not yet implemented[/yellow]")
return 0
return 0
+93 -33
View File
@@ -313,6 +313,18 @@ class PraisonAI:
elif hasattr(args, 'direct_prompt') and args.direct_prompt:
# Only handle direct prompt if agent_file wasn't explicitly set in constructor
if original_agent_file == "agents.yaml": # Default value, so safe to use direct prompt
# Handle --compare flag for CLI mode comparison
if hasattr(args, 'compare') and args.compare:
from .features.compare import CompareHandler
handler = CompareHandler(verbose=getattr(args, 'verbose', False))
result = handler.execute(
args.direct_prompt,
args.compare,
model=getattr(args, 'llm', None),
output_path=getattr(args, 'compare_output', None)
)
return result
# Combine direct prompt with any available inputs (stdin and/or file)
prompt_parts = [args.direct_prompt]
if stdin_input:
@@ -649,7 +661,7 @@ class PraisonAI:
return default_args
# Define special commands
special_commands = ['chat', 'code', 'call', 'realtime', 'train', 'ui', 'context', 'research', 'memory', 'rules', 'workflow', 'hooks', 'knowledge', 'session', 'tools', 'todo', 'docs', 'mcp', 'commit', 'serve', 'schedule', 'skills']
special_commands = ['chat', 'code', 'call', 'realtime', 'train', 'ui', 'context', 'research', 'memory', 'rules', 'workflow', 'hooks', 'knowledge', 'session', 'tools', 'todo', 'docs', 'mcp', 'commit', 'serve', 'schedule', 'skills', 'profile', 'eval']
parser = argparse.ArgumentParser(prog="praisonai", description="praisonAI command-line interface")
parser.add_argument("--framework", choices=["crewai", "autogen", "praisonai"], help="Specify the framework")
@@ -721,8 +733,12 @@ class PraisonAI:
# Metrics - token/cost tracking
parser.add_argument("--metrics", action="store_true", help="Display token usage and cost metrics")
# Image - vision processing
parser.add_argument("--image", type=str, help="Path to image file for vision-based tasks")
# Image Description (Vision) - analyze existing images
parser.add_argument("--image", type=str, help="Path to image file for vision-based description/analysis")
# Image Generation - create new images from text
parser.add_argument("--image-generate", action="store_true", dest="image_generate",
help="Generate an image from the text prompt (use with --llm for model selection)")
# Telemetry - usage monitoring
parser.add_argument("--telemetry", action="store_true", help="Enable usage monitoring and analytics")
@@ -1094,6 +1110,18 @@ class PraisonAI:
exit_code = handle_skills_command(skills_args)
sys.exit(exit_code)
elif args.command == 'profile':
# Profile command - performance profiling and benchmarking
from .features.profile import handle_profile_command
exit_code = handle_profile_command(unknown_args)
sys.exit(exit_code)
elif args.command == 'eval':
# Eval command - evaluate model responses against expected outputs
from .features.eval import handle_eval_command
exit_code = handle_eval_command(unknown_args)
sys.exit(exit_code)
# Only check framework availability for agent-related operations
if not args.command and (args.init or args.auto or args.framework):
@@ -3178,10 +3206,9 @@ Provide ONLY the commit message, no explanations."""
selected_model = router.select_model(prompt, provider)
agent_config["llm"] = selected_model
# Metrics - Token/cost tracking
# Metrics - Token/cost tracking (display happens AFTER execution)
if getattr(self.args, 'metrics', False):
agent_config["metrics"] = True
print("[bold cyan]Metrics enabled - will display token usage and costs[/bold cyan]")
# Telemetry - Usage monitoring
if getattr(self.args, 'telemetry', False):
@@ -3189,6 +3216,12 @@ Provide ONLY the commit message, no explanations."""
telemetry = TelemetryHandler(verbose=getattr(self.args, 'verbose', False))
telemetry.enable()
# Sandbox - Secure command execution (display only, actual sandboxing is in tool approval)
sandbox_mode = getattr(self.args, 'sandbox', None)
if sandbox_mode and sandbox_mode != 'off':
print(f"[bold green]🔒 Sandbox Mode: {sandbox_mode.upper()}[/bold green]")
print("[dim]Commands will be validated before execution[/dim]")
# Auto Memory - Automatic memory extraction (handled post-processing, not as Agent param)
if getattr(self.args, 'auto_memory', False):
print("[bold cyan]Auto Memory enabled - will extract and store memories[/bold cyan]")
@@ -3279,7 +3312,7 @@ Provide ONLY the commit message, no explanations."""
return result
# Image processing - Use ImageAgent instead
# Image Description (Vision) - analyze existing images
if hasattr(self, 'args') and getattr(self.args, 'image', None):
from .features.image import ImageHandler
image_handler = ImageHandler(verbose=getattr(self.args, 'verbose', False))
@@ -3301,6 +3334,27 @@ Provide ONLY the commit message, no explanations."""
return result
# Image Generation - create new images from text
if hasattr(self, 'args') and getattr(self.args, 'image_generate', False):
from .features.image import ImageGenerateHandler
image_gen_handler = ImageGenerateHandler(verbose=getattr(self.args, 'verbose', False))
result = image_gen_handler.execute(
prompt=prompt,
llm=agent_config.get('llm')
)
# Format output for display
if isinstance(result, dict):
if 'error' in result:
print(f"[red]Error: {result['error']}[/red]")
elif 'data' in result and len(result['data']) > 0:
image_url = result['data'][0].get('url', result['data'][0].get('b64_json', 'Generated'))
print(f"[green]Image generated successfully![/green]")
if 'url' in result['data'][0]:
print(f"URL: {result['data'][0]['url']}")
return result
# Flow Display - Visual workflow tracking
if hasattr(self, 'args') and getattr(self.args, 'flow_display', False):
from .features.flow_display import FlowDisplayHandler
@@ -3347,6 +3401,12 @@ Provide ONLY the commit message, no explanations."""
todo = TodoHandler(verbose=getattr(self.args, 'verbose', False))
todo.post_process_result(result, True)
# Telemetry - Display usage summary after execution
if hasattr(self, 'args') and getattr(self.args, 'telemetry', False):
from .features.telemetry import TelemetryHandler
telemetry = TelemetryHandler(verbose=getattr(self.args, 'verbose', False))
telemetry.post_process_result(result, True)
# Flow Display - End workflow
if hasattr(self, 'args') and getattr(self.args, 'flow_display', False):
from .features.flow_display import FlowDisplayHandler
@@ -4741,16 +4801,14 @@ Provide a concise summary (max 200 words):"""
def _process_interactive_prompt_async(self, prompt, tools_list, console, session_state=None):
"""
Process a prompt asynchronously with live status display.
Process a prompt asynchronously with non-blocking status display.
Shows real-time status (tools, commands) while processing in background.
Shows status updates without blocking user input.
Allows user to queue more messages while processing.
"""
import threading
import time
from rich.live import Live
from rich.panel import Panel
from rich.text import Text
import sys
from praisonai.cli.features.message_queue import ProcessingState
state_manager = session_state['state_manager']
@@ -4762,8 +4820,11 @@ Provide a concise summary (max 200 words):"""
live_status.clear()
live_status.update_status("Thinking...")
# Print initial status (non-blocking - user can still type)
console.print(f"[dim cyan]⏳ {live_status.current_status}[/dim cyan]")
# Result container for thread communication
result_container = {'response': None, 'error': None, 'done': False}
result_container = {'response': None, 'error': None, 'done': False, 'last_status': ''}
def process_in_background():
"""Run agent processing in background thread."""
@@ -4809,37 +4870,36 @@ Provide a concise summary (max 200 words):"""
bg_thread = threading.Thread(target=process_in_background, daemon=True)
bg_thread.start()
# Show live status while processing
try:
with Live(console=console, refresh_per_second=4, transient=True) as live:
while not result_container['done']:
# Build status display
status_text = Text()
status_text.append("⏳ ", style="cyan")
status_text.append(live_status.current_status or "Processing...", style="dim")
# Show queued messages if any
queue_count = message_queue.count
if queue_count > 0:
status_text.append(f"\n📋 Queued: {queue_count}", style="dim yellow")
live.update(Panel(status_text, border_style="dim"))
time.sleep(0.1)
except KeyboardInterrupt:
console.print("\n[dim]Processing interrupted[/dim]")
# Wait for completion with periodic status updates (non-blocking style)
start_time = time.time()
last_status_update = ""
while not result_container['done']:
time.sleep(0.2)
current_status = live_status.current_status
# Only print status if it changed
if current_status != last_status_update:
# Use carriage return to update in place
elapsed = time.time() - start_time
queue_count = message_queue.count
queue_info = f" | 📋 Queued: {queue_count}" if queue_count > 0 else ""
sys.stdout.write(f"\r[dim cyan]⏳ {current_status} ({elapsed:.1f}s){queue_info}[/dim cyan]")
sys.stdout.flush()
last_status_update = current_status
# Clear the status line
sys.stdout.write("\r" + " " * 80 + "\r")
sys.stdout.flush()
# Wait for thread to finish
bg_thread.join(timeout=1.0)
# Handle result
if result_container['error']:
console.print(f"\n[red]Error: {result_container['error']}[/red]")
console.print(f"[red]Error: {result_container['error']}[/red]")
elif result_container['response']:
# Print response with streaming effect
console.print()
response_str = result_container['response']
words = response_str.split()
import sys
for i, word in enumerate(words):
console.print(word + " ", end="")
sys.stdout.flush()
+1 -1
View File
@@ -57,7 +57,7 @@ class CloudDeployer:
file.write("FROM python:3.11-slim\n")
file.write("WORKDIR /app\n")
file.write("COPY . .\n")
file.write("RUN pip install flask praisonai==2.3.76 gunicorn markdown\n")
file.write("RUN pip install flask praisonai==2.3.81 gunicorn markdown\n")
file.write("EXPOSE 8080\n")
file.write('CMD ["gunicorn", "-b", "0.0.0.0:8080", "api:app"]\n')
+1 -1
View File
@@ -1 +1 @@
__version__ = "2.3.76"
__version__ = "2.3.81"