Release v2.3.80

docker/Dockerfile.chat

@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
 # Install Python packages (using latest versions)
 RUN pip install --no-cache-dir \
     praisonai_tools \
-    "praisonai>=2.3.79" \
+    "praisonai>=2.3.80" \
     "praisonai[chat]" \
     "embedchain[github,youtube]"
 

docker/Dockerfile.dev

@@ -20,7 +20,7 @@ RUN mkdir -p /root/.praison
 # Install Python packages (using latest versions)
 RUN pip install --no-cache-dir \
     praisonai_tools \
-    "praisonai>=2.3.79" \
+    "praisonai>=2.3.80" \
     "praisonai[ui]" \
     "praisonai[chat]" \
     "praisonai[realtime]" \

docker/Dockerfile.ui

@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
 # Install Python packages (using latest versions)
 RUN pip install --no-cache-dir \
     praisonai_tools \
-    "praisonai>=2.3.79" \
+    "praisonai>=2.3.80" \
     "praisonai[ui]" \
     "praisonai[crewai]"
 

src/praisonai/praisonai.rb

@@ -3,8 +3,8 @@ class Praisonai < Formula
   
     desc "AI tools for various AI applications"
     homepage "https://github.com/MervinPraison/PraisonAI"
-    url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.79.tar.gz"
-    sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.79.tar.gz | shasum -a 256`.split.first
+    url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.80.tar.gz"
+    sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.80.tar.gz | shasum -a 256`.split.first
     license "MIT"
   
     depends_on "[email protected]"

src/praisonai/praisonai/cli/features/eval.py

@@ -347,13 +347,36 @@ class EvalHandler:
         return batch_result
 
 
-def handle_eval_command(args) -> None:
+def handle_eval_command(args) -> int:
     """
     Handle the eval CLI command.
     
     Args:
-        args: Parsed command line arguments
+        args: Command line arguments (list or parsed namespace)
+        
+    Returns:
+        Exit code
     """
+    import argparse
+    
+    # If args is a list, parse it first
+    if isinstance(args, list):
+        parser = argparse.ArgumentParser(prog="praisonai eval")
+        subparsers = parser.add_subparsers(dest='eval_type')
+        add_eval_parser_subcommands(subparsers)
+        
+        try:
+            args = parser.parse_args(args)
+        except SystemExit:
+            return 1
+        
+        if not args.eval_type:
+            parser.print_help()
+            print("\n[bold]Examples:[/bold]")
+            print("  praisonai eval accuracy --prompt \"What is 2+2?\" --expected \"4\"")
+            print("  praisonai eval performance --agent agents.yaml --input \"Hello\"")
+            return 0
+    
     handler = EvalHandler(verbose=getattr(args, 'verbose', False))
     
     eval_type = getattr(args, 'eval_type', 'accuracy')
@@ -420,12 +443,40 @@ def handle_eval_command(args) -> None:
         )
     else:
         print(f"Unknown evaluation type: {eval_type}")
-        return
+        return 1
     
     if 'error' in result:
         print(f"Error: {result['error']}")
+        return 1
     elif not getattr(args, 'quiet', False):
         print(json.dumps(result, indent=2))
+    
+    return 0
+
+
+def add_eval_parser_subcommands(subparsers) -> None:
+    """Add eval subcommand parsers to an existing subparsers object."""
+    accuracy_parser = subparsers.add_parser('accuracy', help='Run accuracy evaluation')
+    accuracy_parser.add_argument('--agent', '-a', help='Agent config file (optional if --prompt used)')
+    accuracy_parser.add_argument('--prompt', '-p', type=str, help='Direct prompt (alternative to --agent)')
+    accuracy_parser.add_argument('--llm', help='LLM model for agent (when using --prompt)')
+    accuracy_parser.add_argument('--input', '-i', help='Input text (defaults to --prompt if not provided)')
+    accuracy_parser.add_argument('--expected', '-e', required=True, help='Expected output')
+    accuracy_parser.add_argument('--iterations', '-n', type=int, default=1, help='Number of iterations')
+    accuracy_parser.add_argument('--model', '-m', help='Judge model')
+    accuracy_parser.add_argument('--output', '-o', help='Output file')
+    accuracy_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
+    accuracy_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
+    
+    perf_parser = subparsers.add_parser('performance', help='Run performance evaluation')
+    perf_parser.add_argument('--agent', '-a', default='agents.yaml', help='Agent config file')
+    perf_parser.add_argument('--input', '-i', default='Hello', help='Input text')
+    perf_parser.add_argument('--iterations', '-n', type=int, default=10, help='Number of iterations')
+    perf_parser.add_argument('--warmup', '-w', type=int, default=2, help='Warmup runs')
+    perf_parser.add_argument('--memory', action='store_true', default=True, help='Track memory')
+    perf_parser.add_argument('--output', '-o', help='Output file')
+    perf_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
+    perf_parser.add_argument('--quiet', '-q', action='store_true', help='Suppress JSON output')
 
 
 def add_eval_parser(subparsers) -> None:

src/praisonai/praisonai/cli/features/metrics.py

@@ -125,11 +125,39 @@ class MetricsHandler(FlagHandler):
         if hasattr(agent, 'metrics') and isinstance(agent.metrics, dict):
             metrics.update(agent.metrics or {})
         
-        # Try to get from litellm's global tracking
+        # Try to get from litellm's success callback data
         try:
             import litellm
-            if hasattr(litellm, '_current_cost'):
-                metrics['cost'] = litellm._current_cost
+            # Check if there's accumulated usage data
+            if hasattr(litellm, '_thread_context') and litellm._thread_context:
+                ctx = litellm._thread_context
+                if hasattr(ctx, 'usage'):
+                    usage = ctx.usage
+                    if hasattr(usage, 'prompt_tokens'):
+                        metrics['prompt_tokens'] = usage.prompt_tokens
+                    if hasattr(usage, 'completion_tokens'):
+                        metrics['completion_tokens'] = usage.completion_tokens
+                    if hasattr(usage, 'total_tokens'):
+                        metrics['total_tokens'] = usage.total_tokens
+        except:
+            pass
+        
+        # Try to get cost from litellm cost tracking
+        try:
+            import litellm
+            from litellm import completion_cost
+            # If we have token counts, estimate cost
+            if 'prompt_tokens' in metrics and 'completion_tokens' in metrics:
+                model = metrics.get('model', 'gpt-4o-mini')
+                try:
+                    cost = completion_cost(
+                        model=model,
+                        prompt_tokens=metrics['prompt_tokens'],
+                        completion_tokens=metrics['completion_tokens']
+                    )
+                    metrics['cost'] = cost
+                except:
+                    pass
         except:
             pass
         

src/praisonai/praisonai/cli/features/profile.py

@@ -0,0 +1,226 @@
+"""
+Profile Handler for CLI.
+
+Provides performance profiling and benchmarking capabilities.
+Usage: praisonai profile <subcommand> [options]
+"""
+
+import argparse
+import time
+import json
+from typing import Any, Dict, List, Optional
+from dataclasses import dataclass, field
+from datetime import datetime
+
+
+@dataclass
+class ProfileResult:
+    """Result from a profiling run."""
+    prompt: str
+    response: str
+    execution_time_ms: float
+    model: str
+    tokens: Optional[Dict[str, int]] = None
+    timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
+    
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "prompt": self.prompt,
+            "response": self.response[:200] + "..." if len(self.response) > 200 else self.response,
+            "execution_time_ms": self.execution_time_ms,
+            "model": self.model,
+            "tokens": self.tokens,
+            "timestamp": self.timestamp
+        }
+
+
+def run_benchmark(prompt: str, model: str = None, iterations: int = 1, verbose: bool = False) -> List[ProfileResult]:
+    """
+    Run a benchmark on a prompt.
+    
+    Args:
+        prompt: The prompt to benchmark
+        model: Optional model to use
+        iterations: Number of iterations to run
+        verbose: Enable verbose output
+        
+    Returns:
+        List of ProfileResult objects
+    """
+    try:
+        from praisonaiagents import Agent
+    except ImportError:
+        print("[red]ERROR: praisonaiagents not installed. Install with: pip install praisonaiagents[/red]")
+        return []
+    
+    results = []
+    
+    agent_config = {
+        "name": "BenchmarkAgent",
+        "role": "Benchmark Assistant",
+        "goal": "Complete tasks efficiently",
+        "verbose": verbose
+    }
+    
+    if model:
+        agent_config["llm"] = model
+    
+    agent = Agent(**agent_config)
+    
+    for i in range(iterations):
+        if verbose:
+            print(f"[dim]Running iteration {i + 1}/{iterations}...[/dim]")
+        
+        start_time = time.time()
+        response = agent.start(prompt)
+        end_time = time.time()
+        
+        execution_time_ms = (end_time - start_time) * 1000
+        
+        result = ProfileResult(
+            prompt=prompt,
+            response=str(response),
+            execution_time_ms=execution_time_ms,
+            model=model or "default"
+        )
+        results.append(result)
+    
+    return results
+
+
+def display_benchmark_results(results: List[ProfileResult], output_format: str = "table"):
+    """Display benchmark results."""
+    if not results:
+        print("[yellow]No results to display[/yellow]")
+        return
+    
+    try:
+        from rich.console import Console
+        from rich.table import Table
+        console = Console()
+    except ImportError:
+        # Fallback to simple output
+        for r in results:
+            print(f"Time: {r.execution_time_ms:.1f}ms | Model: {r.model}")
+        return
+    
+    if output_format == "json":
+        print(json.dumps([r.to_dict() for r in results], indent=2))
+        return
+    
+    # Calculate statistics
+    times = [r.execution_time_ms for r in results]
+    avg_time = sum(times) / len(times)
+    min_time = min(times)
+    max_time = max(times)
+    
+    table = Table(title="📊 Benchmark Results")
+    table.add_column("Metric", style="cyan")
+    table.add_column("Value", style="green")
+    
+    table.add_row("Iterations", str(len(results)))
+    table.add_row("Average Time", f"{avg_time:.1f}ms")
+    table.add_row("Min Time", f"{min_time:.1f}ms")
+    table.add_row("Max Time", f"{max_time:.1f}ms")
+    table.add_row("Model", results[0].model)
+    
+    console.print(table)
+    
+    # Show response preview
+    if results:
+        console.print("\n[bold]Response Preview:[/bold]")
+        preview = results[0].response[:300] + "..." if len(results[0].response) > 300 else results[0].response
+        console.print(f"[dim]{preview}[/dim]")
+
+
+def handle_profile_command(args: List[str]) -> int:
+    """
+    Handle the profile command.
+    
+    Args:
+        args: Command line arguments
+        
+    Returns:
+        Exit code
+    """
+    parser = argparse.ArgumentParser(prog="praisonai profile", description="Performance profiling and benchmarking")
+    subparsers = parser.add_subparsers(dest='subcommand', help='Profile subcommands')
+    
+    # Benchmark subcommand
+    bench_parser = subparsers.add_parser('benchmark', aliases=['bench', 'run'], help='Run a benchmark')
+    bench_parser.add_argument('prompt', nargs='?', help='Prompt to benchmark')
+    bench_parser.add_argument('--model', '-m', type=str, help='Model to use')
+    bench_parser.add_argument('--iterations', '-n', type=int, default=1, help='Number of iterations')
+    bench_parser.add_argument('--output', '-o', type=str, help='Output file path')
+    bench_parser.add_argument('--format', '-f', type=str, choices=['table', 'json', 'html'], default='table', help='Output format')
+    bench_parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
+    
+    # Compare subcommand
+    compare_parser = subparsers.add_parser('compare', help='Compare models')
+    compare_parser.add_argument('prompt', nargs='?', help='Prompt to compare')
+    compare_parser.add_argument('--models', type=str, help='Comma-separated list of models to compare')
+    
+    # Report subcommand
+    report_parser = subparsers.add_parser('report', help='Generate a profiling report')
+    report_parser.add_argument('--input', '-i', type=str, help='Input JSON file with results')
+    report_parser.add_argument('--output', '-o', type=str, help='Output file path')
+    report_parser.add_argument('--format', '-f', type=str, choices=['html', 'json', 'markdown'], default='html', help='Report format')
+    
+    try:
+        parsed_args = parser.parse_args(args)
+    except SystemExit:
+        return 1
+    
+    if not parsed_args.subcommand:
+        parser.print_help()
+        print("\n[bold]Examples:[/bold]")
+        print("  praisonai profile benchmark \"What is 2+2?\"")
+        print("  praisonai profile benchmark \"Write a poem\" --model gpt-4o --iterations 3")
+        print("  praisonai profile compare \"Explain AI\" --models gpt-4o,gpt-4o-mini")
+        return 0
+    
+    if parsed_args.subcommand in ['benchmark', 'bench', 'run']:
+        if not parsed_args.prompt:
+            print("[red]ERROR: Please provide a prompt to benchmark[/red]")
+            print("Usage: praisonai profile benchmark \"Your prompt here\"")
+            return 1
+        
+        print(f"[bold cyan]🔬 Running benchmark...[/bold cyan]")
+        results = run_benchmark(
+            parsed_args.prompt,
+            model=parsed_args.model,
+            iterations=parsed_args.iterations,
+            verbose=parsed_args.verbose
+        )
+        
+        display_benchmark_results(results, parsed_args.format)
+        
+        if parsed_args.output:
+            with open(parsed_args.output, 'w') as f:
+                json.dump([r.to_dict() for r in results], f, indent=2)
+            print(f"[green]Results saved to {parsed_args.output}[/green]")
+        
+        return 0
+    
+    elif parsed_args.subcommand == 'compare':
+        if not parsed_args.prompt:
+            print("[red]ERROR: Please provide a prompt to compare[/red]")
+            return 1
+        
+        models = parsed_args.models.split(',') if parsed_args.models else ['gpt-4o-mini', 'gpt-4o']
+        print(f"[bold cyan]🔬 Comparing models: {', '.join(models)}[/bold cyan]")
+        
+        all_results = []
+        for model in models:
+            print(f"[dim]Testing {model}...[/dim]")
+            results = run_benchmark(parsed_args.prompt, model=model, iterations=1)
+            all_results.extend(results)
+        
+        display_benchmark_results(all_results)
+        return 0
+    
+    elif parsed_args.subcommand == 'report':
+        print("[yellow]Report generation not yet implemented[/yellow]")
+        return 0
+    
+    return 0

src/praisonai/praisonai/cli/main.py

@@ -661,7 +661,7 @@ class PraisonAI:
             return default_args
         
         # Define special commands
-        special_commands = ['chat', 'code', 'call', 'realtime', 'train', 'ui', 'context', 'research', 'memory', 'rules', 'workflow', 'hooks', 'knowledge', 'session', 'tools', 'todo', 'docs', 'mcp', 'commit', 'serve', 'schedule', 'skills']
+        special_commands = ['chat', 'code', 'call', 'realtime', 'train', 'ui', 'context', 'research', 'memory', 'rules', 'workflow', 'hooks', 'knowledge', 'session', 'tools', 'todo', 'docs', 'mcp', 'commit', 'serve', 'schedule', 'skills', 'profile', 'eval']
         
         parser = argparse.ArgumentParser(prog="praisonai", description="praisonAI command-line interface")
         parser.add_argument("--framework", choices=["crewai", "autogen", "praisonai"], help="Specify the framework")
@@ -1110,6 +1110,18 @@ class PraisonAI:
                 
                 exit_code = handle_skills_command(skills_args)
                 sys.exit(exit_code)
+            
+            elif args.command == 'profile':
+                # Profile command - performance profiling and benchmarking
+                from .features.profile import handle_profile_command
+                exit_code = handle_profile_command(unknown_args)
+                sys.exit(exit_code)
+            
+            elif args.command == 'eval':
+                # Eval command - evaluate model responses against expected outputs
+                from .features.eval import handle_eval_command
+                exit_code = handle_eval_command(unknown_args)
+                sys.exit(exit_code)
 
         # Only check framework availability for agent-related operations
         if not args.command and (args.init or args.auto or args.framework):
@@ -3194,10 +3206,9 @@ Provide ONLY the commit message, no explanations."""
                     selected_model = router.select_model(prompt, provider)
                     agent_config["llm"] = selected_model
                 
-                # Metrics - Token/cost tracking
+                # Metrics - Token/cost tracking (display happens AFTER execution)
                 if getattr(self.args, 'metrics', False):
                     agent_config["metrics"] = True
-                    print("[bold cyan]Metrics enabled - will display token usage and costs[/bold cyan]")
                 
                 # Telemetry - Usage monitoring
                 if getattr(self.args, 'telemetry', False):
@@ -3205,6 +3216,12 @@ Provide ONLY the commit message, no explanations."""
                     telemetry = TelemetryHandler(verbose=getattr(self.args, 'verbose', False))
                     telemetry.enable()
                 
+                # Sandbox - Secure command execution (display only, actual sandboxing is in tool approval)
+                sandbox_mode = getattr(self.args, 'sandbox', None)
+                if sandbox_mode and sandbox_mode != 'off':
+                    print(f"[bold green]🔒 Sandbox Mode: {sandbox_mode.upper()}[/bold green]")
+                    print("[dim]Commands will be validated before execution[/dim]")
+                
                 # Auto Memory - Automatic memory extraction (handled post-processing, not as Agent param)
                 if getattr(self.args, 'auto_memory', False):
                     print("[bold cyan]Auto Memory enabled - will extract and store memories[/bold cyan]")
@@ -3384,6 +3401,12 @@ Provide ONLY the commit message, no explanations."""
                 todo = TodoHandler(verbose=getattr(self.args, 'verbose', False))
                 todo.post_process_result(result, True)
             
+            # Telemetry - Display usage summary after execution
+            if hasattr(self, 'args') and getattr(self.args, 'telemetry', False):
+                from .features.telemetry import TelemetryHandler
+                telemetry = TelemetryHandler(verbose=getattr(self.args, 'verbose', False))
+                telemetry.post_process_result(result, True)
+            
             # Flow Display - End workflow
             if hasattr(self, 'args') and getattr(self.args, 'flow_display', False):
                 from .features.flow_display import FlowDisplayHandler

src/praisonai/praisonai/deploy.py

@@ -57,7 +57,7 @@ class CloudDeployer:
             file.write("FROM python:3.11-slim\n")
             file.write("WORKDIR /app\n")
             file.write("COPY . .\n")
-            file.write("RUN pip install flask praisonai==2.3.79 gunicorn markdown\n")
+            file.write("RUN pip install flask praisonai==2.3.80 gunicorn markdown\n")
             file.write("EXPOSE 8080\n")
             file.write('CMD ["gunicorn", "-b", "0.0.0.0:8080", "api:app"]\n')
             

src/praisonai/praisonai/version.py

@@ -1 +1 @@
-__version__ = "2.3.79"
+__version__ = "2.3.80"