mirror of
https://github.com/MervinPraison/PraisonAI.git
synced 2026-11-08 17:31:00 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5a13571fdd |
No files matched your search
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=2.3.74" \
|
||||
"praisonai>=2.3.75" \
|
||||
"praisonai[chat]" \
|
||||
"embedchain[github,youtube]"
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=2.3.74" \
|
||||
"praisonai>=2.3.75" \
|
||||
"praisonai[ui]" \
|
||||
"praisonai[chat]" \
|
||||
"praisonai[realtime]" \
|
||||
|
||||
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=2.3.74" \
|
||||
"praisonai>=2.3.75" \
|
||||
"praisonai[ui]" \
|
||||
"praisonai[crewai]"
|
||||
|
||||
|
||||
@@ -3,8 +3,8 @@ class Praisonai < Formula
|
||||
|
||||
desc "AI tools for various AI applications"
|
||||
homepage "https://github.com/MervinPraison/PraisonAI"
|
||||
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.74.tar.gz"
|
||||
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.74.tar.gz | shasum -a 256`.split.first
|
||||
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.75.tar.gz"
|
||||
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.75.tar.gz | shasum -a 256`.split.first
|
||||
license "MIT"
|
||||
|
||||
depends_on "[email protected]"
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
"""
|
||||
PraisonAI CLI Profiling Commands
|
||||
|
||||
Provides CLI commands for profiling agent execution.
|
||||
|
||||
Usage:
|
||||
praisonai profile run "task" --output report.html
|
||||
praisonai profile report --format json
|
||||
praisonai profile benchmark --iterations 10
|
||||
"""
|
||||
|
||||
import time
|
||||
import json
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
|
||||
class ProfilingHandler:
|
||||
"""
|
||||
Handler for CLI profiling commands.
|
||||
|
||||
Features:
|
||||
- Run tasks with profiling enabled
|
||||
- Generate reports (console, JSON, HTML)
|
||||
- Benchmark agent performance
|
||||
- Export flamegraphs
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._profiler = None
|
||||
|
||||
@property
|
||||
def profiler(self):
|
||||
"""Lazy load profiler."""
|
||||
if self._profiler is None:
|
||||
from praisonai.profiler import Profiler
|
||||
self._profiler = Profiler
|
||||
return self._profiler
|
||||
|
||||
def enable(self) -> None:
|
||||
"""Enable profiling."""
|
||||
self.profiler.enable()
|
||||
|
||||
def disable(self) -> None:
|
||||
"""Disable profiling."""
|
||||
self.profiler.disable()
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear profiling data."""
|
||||
self.profiler.clear()
|
||||
|
||||
def run_with_profiling(self, task: str, config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Run a task with profiling enabled.
|
||||
|
||||
Args:
|
||||
task: Task description or prompt
|
||||
config: Optional configuration dict
|
||||
|
||||
Returns:
|
||||
Dict with result and profiling data
|
||||
"""
|
||||
self.profiler.enable()
|
||||
self.profiler.clear()
|
||||
|
||||
result = None
|
||||
error = None
|
||||
|
||||
try:
|
||||
# Import and run PraisonAI
|
||||
from praisonai import PraisonAI
|
||||
|
||||
with self.profiler.block("total_execution"):
|
||||
with self.profiler.block("initialization"):
|
||||
praison = PraisonAI(auto=task, **(config or {}))
|
||||
|
||||
with self.profiler.block("agent_run"):
|
||||
result = praison.run()
|
||||
except Exception as e:
|
||||
error = str(e)
|
||||
finally:
|
||||
self.profiler.disable()
|
||||
|
||||
return {
|
||||
'result': result,
|
||||
'error': error,
|
||||
'summary': self.profiler.get_summary(),
|
||||
'statistics': self.profiler.get_statistics()
|
||||
}
|
||||
|
||||
def report(self, format: str = "console", output: Optional[str] = None) -> str:
|
||||
"""
|
||||
Generate profiling report.
|
||||
|
||||
Args:
|
||||
format: Output format (console, json, html)
|
||||
output: Optional file path to save report
|
||||
|
||||
Returns:
|
||||
Report content as string
|
||||
"""
|
||||
if format == "console":
|
||||
content = self.profiler.report(output="console")
|
||||
elif format == "json":
|
||||
content = self.profiler.export_json()
|
||||
elif format == "html":
|
||||
content = self.profiler.export_html()
|
||||
else:
|
||||
raise ValueError(f"Unknown format: {format}")
|
||||
|
||||
if output:
|
||||
with open(output, 'w') as f:
|
||||
f.write(content)
|
||||
print(f"Report saved to: {output}")
|
||||
|
||||
return content
|
||||
|
||||
def benchmark(self, task: str, iterations: int = 5,
|
||||
warmup: int = 1, config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Benchmark agent performance.
|
||||
|
||||
Args:
|
||||
task: Task to benchmark
|
||||
iterations: Number of iterations
|
||||
warmup: Number of warmup runs
|
||||
config: Optional configuration
|
||||
|
||||
Returns:
|
||||
Benchmark results with statistics
|
||||
"""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
times = []
|
||||
errors = []
|
||||
|
||||
# Warmup runs
|
||||
for i in range(warmup):
|
||||
print(f"Warmup {i + 1}/{warmup}...")
|
||||
try:
|
||||
self.run_with_profiling(task, config)
|
||||
except Exception as e:
|
||||
print(f"Warmup error: {e}")
|
||||
|
||||
# Benchmark runs
|
||||
for i in range(iterations):
|
||||
print(f"Iteration {i + 1}/{iterations}...")
|
||||
Profiler.clear()
|
||||
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
self.run_with_profiling(task, config)
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
times.append(duration_ms)
|
||||
except Exception as e:
|
||||
errors.append(str(e))
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
times.append(duration_ms)
|
||||
|
||||
# Calculate statistics
|
||||
if times:
|
||||
sorted_times = sorted(times)
|
||||
n = len(sorted_times)
|
||||
|
||||
results = {
|
||||
'iterations': iterations,
|
||||
'successful': iterations - len(errors),
|
||||
'failed': len(errors),
|
||||
'times_ms': times,
|
||||
'mean_ms': sum(times) / n,
|
||||
'min_ms': min(times),
|
||||
'max_ms': max(times),
|
||||
'p50_ms': sorted_times[n // 2],
|
||||
'p95_ms': sorted_times[int(n * 0.95)] if n > 1 else sorted_times[-1],
|
||||
'p99_ms': sorted_times[int(n * 0.99)] if n > 1 else sorted_times[-1],
|
||||
'errors': errors
|
||||
}
|
||||
else:
|
||||
results = {
|
||||
'iterations': iterations,
|
||||
'successful': 0,
|
||||
'failed': len(errors),
|
||||
'errors': errors
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
def export_flamegraph(self, filepath: str) -> None:
|
||||
"""Export flamegraph to SVG file."""
|
||||
self.profiler.export_flamegraph(filepath)
|
||||
print(f"Flamegraph saved to: {filepath}")
|
||||
|
||||
def get_summary(self) -> Dict[str, Any]:
|
||||
"""Get profiling summary."""
|
||||
return self.profiler.get_summary()
|
||||
|
||||
def get_statistics(self, category: Optional[str] = None) -> Dict[str, float]:
|
||||
"""Get profiling statistics."""
|
||||
return self.profiler.get_statistics(category)
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print profiling summary to console."""
|
||||
summary = self.get_summary()
|
||||
stats = self.get_statistics()
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("PROFILING SUMMARY")
|
||||
print("=" * 60)
|
||||
|
||||
print(f"\nTotal Time: {summary['total_time_ms']:.2f}ms")
|
||||
print(f"Operations: {summary['timing_count']}")
|
||||
print(f"Imports: {summary['import_count']}")
|
||||
print(f"Flow Steps: {summary['flow_steps']}")
|
||||
|
||||
print("\nStatistics:")
|
||||
print(f" P50 (Median): {stats['p50']:.2f}ms")
|
||||
print(f" P95: {stats['p95']:.2f}ms")
|
||||
print(f" P99: {stats['p99']:.2f}ms")
|
||||
print(f" Mean: {stats['mean']:.2f}ms")
|
||||
print(f" Std Dev: {stats['std_dev']:.2f}ms")
|
||||
|
||||
print("\nSlowest Operations:")
|
||||
for name, duration in summary['slowest_operations'][:5]:
|
||||
print(f" {name}: {duration:.2f}ms")
|
||||
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
def handle_profile_command(args) -> int:
|
||||
"""
|
||||
Handle profile CLI command.
|
||||
|
||||
Args:
|
||||
args: Parsed command line arguments
|
||||
|
||||
Returns:
|
||||
Exit code (0 for success)
|
||||
"""
|
||||
handler = ProfilingHandler()
|
||||
|
||||
subcommand = getattr(args, 'profile_command', None)
|
||||
|
||||
if subcommand == 'run':
|
||||
task = getattr(args, 'task', '')
|
||||
output = getattr(args, 'output', None)
|
||||
format_type = getattr(args, 'format', 'console')
|
||||
|
||||
print(f"Running with profiling: {task[:50]}...")
|
||||
result = handler.run_with_profiling(task)
|
||||
|
||||
if result['error']:
|
||||
print(f"Error: {result['error']}")
|
||||
|
||||
handler.report(format=format_type, output=output)
|
||||
return 0 if not result['error'] else 1
|
||||
|
||||
elif subcommand == 'report':
|
||||
output = getattr(args, 'output', None)
|
||||
format_type = getattr(args, 'format', 'console')
|
||||
|
||||
handler.report(format=format_type, output=output)
|
||||
return 0
|
||||
|
||||
elif subcommand == 'benchmark':
|
||||
task = getattr(args, 'task', '')
|
||||
iterations = getattr(args, 'iterations', 5)
|
||||
warmup = getattr(args, 'warmup', 1)
|
||||
output = getattr(args, 'output', None)
|
||||
|
||||
print(f"Benchmarking: {task[:50]}...")
|
||||
results = handler.benchmark(task, iterations=iterations, warmup=warmup)
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("BENCHMARK RESULTS")
|
||||
print("=" * 60)
|
||||
print(f"Iterations: {results['iterations']}")
|
||||
print(f"Successful: {results['successful']}")
|
||||
print(f"Failed: {results['failed']}")
|
||||
|
||||
if 'mean_ms' in results:
|
||||
print("\nTiming:")
|
||||
print(f" Mean: {results['mean_ms']:.2f}ms")
|
||||
print(f" Min: {results['min_ms']:.2f}ms")
|
||||
print(f" Max: {results['max_ms']:.2f}ms")
|
||||
print(f" P50: {results['p50_ms']:.2f}ms")
|
||||
print(f" P95: {results['p95_ms']:.2f}ms")
|
||||
|
||||
if output:
|
||||
with open(output, 'w') as f:
|
||||
json.dump(results, f, indent=2)
|
||||
print(f"\nResults saved to: {output}")
|
||||
|
||||
return 0
|
||||
|
||||
elif subcommand == 'flamegraph':
|
||||
output = getattr(args, 'output', 'profile.svg')
|
||||
handler.export_flamegraph(output)
|
||||
return 0
|
||||
|
||||
elif subcommand == 'summary':
|
||||
handler.print_summary()
|
||||
return 0
|
||||
|
||||
else:
|
||||
print("Usage: praisonai profile <run|report|benchmark|flamegraph|summary>")
|
||||
return 1
|
||||
|
||||
|
||||
def add_profile_parser(subparsers) -> None:
|
||||
"""Add profile subcommand to argument parser."""
|
||||
profile_parser = subparsers.add_parser(
|
||||
'profile',
|
||||
help='Profile agent execution'
|
||||
)
|
||||
|
||||
profile_subparsers = profile_parser.add_subparsers(dest='profile_command')
|
||||
|
||||
# Run with profiling
|
||||
run_parser = profile_subparsers.add_parser('run', help='Run task with profiling')
|
||||
run_parser.add_argument('task', help='Task to run')
|
||||
run_parser.add_argument('--output', '-o', help='Output file path')
|
||||
run_parser.add_argument('--format', '-f', choices=['console', 'json', 'html'],
|
||||
default='console', help='Output format')
|
||||
|
||||
# Generate report
|
||||
report_parser = profile_subparsers.add_parser('report', help='Generate profiling report')
|
||||
report_parser.add_argument('--output', '-o', help='Output file path')
|
||||
report_parser.add_argument('--format', '-f', choices=['console', 'json', 'html'],
|
||||
default='console', help='Output format')
|
||||
|
||||
# Benchmark
|
||||
bench_parser = profile_subparsers.add_parser('benchmark', help='Benchmark agent performance')
|
||||
bench_parser.add_argument('task', help='Task to benchmark')
|
||||
bench_parser.add_argument('--iterations', '-n', type=int, default=5, help='Number of iterations')
|
||||
bench_parser.add_argument('--warmup', '-w', type=int, default=1, help='Warmup runs')
|
||||
bench_parser.add_argument('--output', '-o', help='Output file for results')
|
||||
|
||||
# Flamegraph
|
||||
flame_parser = profile_subparsers.add_parser('flamegraph', help='Export flamegraph')
|
||||
flame_parser.add_argument('--output', '-o', default='profile.svg', help='Output SVG file')
|
||||
|
||||
# Summary
|
||||
profile_subparsers.add_parser('summary', help='Print profiling summary') # noqa: F841
|
||||
|
||||
|
||||
# Exports
|
||||
__all__ = [
|
||||
'ProfilingHandler',
|
||||
'handle_profile_command',
|
||||
'add_profile_parser',
|
||||
]
|
||||
@@ -57,7 +57,7 @@ class CloudDeployer:
|
||||
file.write("FROM python:3.11-slim\n")
|
||||
file.write("WORKDIR /app\n")
|
||||
file.write("COPY . .\n")
|
||||
file.write("RUN pip install flask praisonai==2.3.74 gunicorn markdown\n")
|
||||
file.write("RUN pip install flask praisonai==2.3.75 gunicorn markdown\n")
|
||||
file.write("EXPOSE 8080\n")
|
||||
file.write('CMD ["gunicorn", "-b", "0.0.0.0:8080", "api:app"]\n')
|
||||
|
||||
|
||||
@@ -8,7 +8,14 @@ Features:
|
||||
- Function execution timing
|
||||
- Flow tracking
|
||||
- File/module usage tracking
|
||||
- Memory usage (optional)
|
||||
- Memory usage (tracemalloc)
|
||||
- API call profiling (wall-clock time)
|
||||
- Streaming profiling (TTFT, total time)
|
||||
- Statistics (p50, p95, p99)
|
||||
- cProfile integration
|
||||
- Flamegraph generation
|
||||
- Line-level profiling
|
||||
- JSON/HTML export
|
||||
|
||||
Usage:
|
||||
from praisonai.profiler import Profiler, profile, profile_imports
|
||||
@@ -22,12 +29,27 @@ Usage:
|
||||
with Profiler.block("my_operation"):
|
||||
do_something()
|
||||
|
||||
# Profile API calls
|
||||
with Profiler.api_call("https://api.example.com") as call:
|
||||
response = requests.get(...)
|
||||
|
||||
# Profile streaming
|
||||
with Profiler.streaming("chat") as tracker:
|
||||
tracker.first_token()
|
||||
for chunk in stream:
|
||||
tracker.chunk()
|
||||
|
||||
# Profile imports
|
||||
with profile_imports():
|
||||
import heavy_module
|
||||
|
||||
# Get report
|
||||
# Get report with statistics
|
||||
Profiler.report()
|
||||
stats = Profiler.get_statistics()
|
||||
|
||||
# Export
|
||||
Profiler.export_json()
|
||||
Profiler.export_html()
|
||||
"""
|
||||
|
||||
import time
|
||||
@@ -35,9 +57,15 @@ import functools
|
||||
import threading
|
||||
import sys
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
import json
|
||||
import tracemalloc
|
||||
import cProfile
|
||||
import pstats
|
||||
import io
|
||||
import statistics
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from typing import Dict, List, Optional, Callable, Any
|
||||
from contextlib import contextmanager
|
||||
from contextlib import contextmanager, asynccontextmanager
|
||||
|
||||
|
||||
# ============================================================================
|
||||
@@ -55,6 +83,38 @@ class TimingRecord:
|
||||
timestamp: float = field(default_factory=time.time)
|
||||
|
||||
|
||||
@dataclass
|
||||
class APICallRecord:
|
||||
"""Record of an API/HTTP call."""
|
||||
endpoint: str
|
||||
method: str
|
||||
duration_ms: float
|
||||
status_code: int = 0
|
||||
request_size: int = 0
|
||||
response_size: int = 0
|
||||
timestamp: float = field(default_factory=time.time)
|
||||
|
||||
|
||||
@dataclass
|
||||
class StreamingRecord:
|
||||
"""Record of streaming operation (LLM responses)."""
|
||||
name: str
|
||||
ttft_ms: float # Time to first token
|
||||
total_ms: float
|
||||
chunk_count: int = 0
|
||||
total_tokens: int = 0
|
||||
timestamp: float = field(default_factory=time.time)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MemoryRecord:
|
||||
"""Record of memory usage."""
|
||||
name: str
|
||||
current_kb: float
|
||||
peak_kb: float
|
||||
timestamp: float = field(default_factory=time.time)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportRecord:
|
||||
"""Record of a module import."""
|
||||
@@ -75,6 +135,80 @@ class FlowRecord:
|
||||
timestamp: float = field(default_factory=time.time)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Streaming Tracker
|
||||
# ============================================================================
|
||||
|
||||
class StreamingTracker:
|
||||
"""
|
||||
Track streaming operations (LLM responses).
|
||||
|
||||
Usage:
|
||||
tracker = StreamingTracker("chat")
|
||||
tracker.start()
|
||||
tracker.first_token() # Mark TTFT
|
||||
for chunk in stream:
|
||||
tracker.chunk()
|
||||
tracker.end(total_tokens=100)
|
||||
"""
|
||||
|
||||
def __init__(self, name: str):
|
||||
self.name = name
|
||||
self._start_time: Optional[float] = None
|
||||
self._first_token_time: Optional[float] = None
|
||||
self._end_time: Optional[float] = None
|
||||
self._chunk_count: int = 0
|
||||
self._total_tokens: int = 0
|
||||
|
||||
def start(self) -> None:
|
||||
"""Start tracking."""
|
||||
self._start_time = time.perf_counter()
|
||||
|
||||
def first_token(self) -> None:
|
||||
"""Mark time to first token."""
|
||||
if self._first_token_time is None:
|
||||
self._first_token_time = time.perf_counter()
|
||||
|
||||
def chunk(self) -> None:
|
||||
"""Record a chunk received."""
|
||||
self._chunk_count += 1
|
||||
|
||||
def end(self, total_tokens: int = 0) -> None:
|
||||
"""End tracking and record to Profiler."""
|
||||
self._end_time = time.perf_counter()
|
||||
self._total_tokens = total_tokens
|
||||
|
||||
if self._start_time is not None:
|
||||
ttft_ms = 0.0
|
||||
if self._first_token_time is not None:
|
||||
ttft_ms = (self._first_token_time - self._start_time) * 1000
|
||||
|
||||
total_ms = (self._end_time - self._start_time) * 1000
|
||||
|
||||
Profiler.record_streaming(
|
||||
name=self.name,
|
||||
ttft_ms=ttft_ms,
|
||||
total_ms=total_ms,
|
||||
chunk_count=self._chunk_count,
|
||||
total_tokens=self._total_tokens
|
||||
)
|
||||
|
||||
@property
|
||||
def ttft_ms(self) -> float:
|
||||
"""Get time to first token in ms."""
|
||||
if self._start_time and self._first_token_time:
|
||||
return (self._first_token_time - self._start_time) * 1000
|
||||
return 0.0
|
||||
|
||||
@property
|
||||
def elapsed_ms(self) -> float:
|
||||
"""Get elapsed time in ms."""
|
||||
if self._start_time:
|
||||
end = self._end_time or time.perf_counter()
|
||||
return (end - self._start_time) * 1000
|
||||
return 0.0
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Profiler Class
|
||||
# ============================================================================
|
||||
@@ -84,6 +218,16 @@ class Profiler:
|
||||
Centralized profiler for performance monitoring.
|
||||
|
||||
Thread-safe singleton pattern for global access.
|
||||
|
||||
Features:
|
||||
- Function/block timing
|
||||
- API call profiling (wall-clock)
|
||||
- Streaming profiling (TTFT)
|
||||
- Memory profiling
|
||||
- Import timing
|
||||
- Statistics (p50, p95, p99)
|
||||
- cProfile integration
|
||||
- Export (JSON, HTML)
|
||||
"""
|
||||
|
||||
_instance: Optional['Profiler'] = None
|
||||
@@ -93,9 +237,14 @@ class Profiler:
|
||||
_timings: List[TimingRecord] = []
|
||||
_imports: List[ImportRecord] = []
|
||||
_flow: List[FlowRecord] = []
|
||||
_api_calls: List[APICallRecord] = []
|
||||
_streaming: List[StreamingRecord] = []
|
||||
_memory: List[MemoryRecord] = []
|
||||
_enabled: bool = False
|
||||
_flow_step: int = 0
|
||||
_files_accessed: Dict[str, int] = {}
|
||||
_line_profile_data: Dict[str, Any] = {}
|
||||
_cprofile_stats: List[Dict[str, Any]] = []
|
||||
|
||||
def __new__(cls):
|
||||
if cls._instance is None:
|
||||
@@ -126,8 +275,13 @@ class Profiler:
|
||||
cls._timings.clear()
|
||||
cls._imports.clear()
|
||||
cls._flow.clear()
|
||||
cls._api_calls.clear()
|
||||
cls._streaming.clear()
|
||||
cls._memory.clear()
|
||||
cls._flow_step = 0
|
||||
cls._files_accessed.clear()
|
||||
cls._line_profile_data.clear()
|
||||
cls._cprofile_stats.clear()
|
||||
|
||||
@classmethod
|
||||
def record_timing(cls, name: str, duration_ms: float, category: str = "function",
|
||||
@@ -295,6 +449,433 @@ class Profiler:
|
||||
print(report_text)
|
||||
|
||||
return report_text
|
||||
|
||||
# ========================================================================
|
||||
# API Call Profiling
|
||||
# ========================================================================
|
||||
|
||||
@classmethod
|
||||
def record_api_call(cls, endpoint: str, method: str, duration_ms: float,
|
||||
status_code: int = 0, request_size: int = 0,
|
||||
response_size: int = 0) -> None:
|
||||
"""Record an API/HTTP call timing."""
|
||||
if not cls.is_enabled():
|
||||
return
|
||||
|
||||
with cls._lock:
|
||||
cls._api_calls.append(APICallRecord(
|
||||
endpoint=endpoint,
|
||||
method=method,
|
||||
duration_ms=duration_ms,
|
||||
status_code=status_code,
|
||||
request_size=request_size,
|
||||
response_size=response_size
|
||||
))
|
||||
|
||||
@classmethod
|
||||
def get_api_calls(cls) -> List[APICallRecord]:
|
||||
"""Get API call records."""
|
||||
with cls._lock:
|
||||
return cls._api_calls.copy()
|
||||
|
||||
@classmethod
|
||||
@contextmanager
|
||||
def api_call(cls, endpoint: str, method: str = "GET"):
|
||||
"""Context manager for profiling API calls."""
|
||||
start = time.perf_counter()
|
||||
call_info = {'status_code': 0, 'request_size': 0, 'response_size': 0}
|
||||
|
||||
try:
|
||||
yield call_info
|
||||
finally:
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
cls.record_api_call(
|
||||
endpoint=endpoint,
|
||||
method=method,
|
||||
duration_ms=duration_ms,
|
||||
status_code=call_info.get('status_code', 0),
|
||||
request_size=call_info.get('request_size', 0),
|
||||
response_size=call_info.get('response_size', 0)
|
||||
)
|
||||
|
||||
# ========================================================================
|
||||
# Streaming Profiling
|
||||
# ========================================================================
|
||||
|
||||
@classmethod
|
||||
def record_streaming(cls, name: str, ttft_ms: float, total_ms: float,
|
||||
chunk_count: int = 0, total_tokens: int = 0) -> None:
|
||||
"""Record streaming metrics."""
|
||||
if not cls.is_enabled():
|
||||
return
|
||||
|
||||
with cls._lock:
|
||||
cls._streaming.append(StreamingRecord(
|
||||
name=name,
|
||||
ttft_ms=ttft_ms,
|
||||
total_ms=total_ms,
|
||||
chunk_count=chunk_count,
|
||||
total_tokens=total_tokens
|
||||
))
|
||||
|
||||
@classmethod
|
||||
def get_streaming_records(cls) -> List[StreamingRecord]:
|
||||
"""Get streaming records."""
|
||||
with cls._lock:
|
||||
return cls._streaming.copy()
|
||||
|
||||
@classmethod
|
||||
@contextmanager
|
||||
def streaming(cls, name: str):
|
||||
"""Context manager for profiling streaming operations."""
|
||||
tracker = StreamingTracker(name)
|
||||
tracker.start()
|
||||
try:
|
||||
yield tracker
|
||||
finally:
|
||||
tracker.end()
|
||||
|
||||
@classmethod
|
||||
@asynccontextmanager
|
||||
async def streaming_async(cls, name: str):
|
||||
"""Async context manager for profiling streaming operations."""
|
||||
tracker = StreamingTracker(name)
|
||||
tracker.start()
|
||||
try:
|
||||
yield tracker
|
||||
finally:
|
||||
tracker.end()
|
||||
|
||||
# ========================================================================
|
||||
# Memory Profiling
|
||||
# ========================================================================
|
||||
|
||||
@classmethod
|
||||
def record_memory(cls, name: str, current_kb: float, peak_kb: float) -> None:
|
||||
"""Record memory usage."""
|
||||
if not cls.is_enabled():
|
||||
return
|
||||
|
||||
with cls._lock:
|
||||
cls._memory.append(MemoryRecord(
|
||||
name=name,
|
||||
current_kb=current_kb,
|
||||
peak_kb=peak_kb
|
||||
))
|
||||
|
||||
@classmethod
|
||||
def get_memory_records(cls) -> List[MemoryRecord]:
|
||||
"""Get memory records."""
|
||||
with cls._lock:
|
||||
return cls._memory.copy()
|
||||
|
||||
@classmethod
|
||||
@contextmanager
|
||||
def memory(cls, name: str):
|
||||
"""Context manager for profiling memory usage."""
|
||||
tracemalloc.start()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
current, peak = tracemalloc.get_traced_memory()
|
||||
tracemalloc.stop()
|
||||
cls.record_memory(name, current / 1024, peak / 1024)
|
||||
|
||||
@classmethod
|
||||
def memory_snapshot(cls) -> Dict[str, float]:
|
||||
"""Take a memory snapshot."""
|
||||
tracemalloc.start()
|
||||
current, peak = tracemalloc.get_traced_memory()
|
||||
tracemalloc.stop()
|
||||
return {
|
||||
'current_kb': current / 1024,
|
||||
'peak_kb': peak / 1024
|
||||
}
|
||||
|
||||
# ========================================================================
|
||||
# Statistics
|
||||
# ========================================================================
|
||||
|
||||
@classmethod
|
||||
def get_statistics(cls, category: Optional[str] = None) -> Dict[str, float]:
|
||||
"""
|
||||
Get statistical analysis of timing data.
|
||||
|
||||
Returns p50, p95, p99, mean, std_dev, min, max.
|
||||
"""
|
||||
with cls._lock:
|
||||
if category:
|
||||
durations = [t.duration_ms for t in cls._timings if t.category == category]
|
||||
else:
|
||||
durations = [t.duration_ms for t in cls._timings]
|
||||
|
||||
if not durations:
|
||||
return {
|
||||
'p50': 0.0, 'p95': 0.0, 'p99': 0.0,
|
||||
'mean': 0.0, 'std_dev': 0.0, 'min': 0.0, 'max': 0.0,
|
||||
'count': 0
|
||||
}
|
||||
|
||||
sorted_durations = sorted(durations)
|
||||
n = len(sorted_durations)
|
||||
|
||||
def percentile(p: float) -> float:
|
||||
idx = int(n * p / 100)
|
||||
return sorted_durations[min(idx, n - 1)]
|
||||
|
||||
mean = statistics.mean(durations)
|
||||
std_dev = statistics.stdev(durations) if n > 1 else 0.0
|
||||
|
||||
return {
|
||||
'p50': percentile(50),
|
||||
'p95': percentile(95),
|
||||
'p99': percentile(99),
|
||||
'mean': mean,
|
||||
'std_dev': std_dev,
|
||||
'min': min(durations),
|
||||
'max': max(durations),
|
||||
'count': n
|
||||
}
|
||||
|
||||
# ========================================================================
|
||||
# cProfile Integration
|
||||
# ========================================================================
|
||||
|
||||
@classmethod
|
||||
@contextmanager
|
||||
def cprofile(cls, name: str):
|
||||
"""Context manager for cProfile profiling."""
|
||||
pr = cProfile.Profile()
|
||||
pr.enable()
|
||||
|
||||
try:
|
||||
yield pr
|
||||
finally:
|
||||
pr.disable()
|
||||
|
||||
# Store stats
|
||||
s = io.StringIO()
|
||||
ps = pstats.Stats(pr, stream=s).sort_stats('cumulative')
|
||||
ps.print_stats(30)
|
||||
|
||||
with cls._lock:
|
||||
cls._cprofile_stats.append({
|
||||
'name': name,
|
||||
'stats': s.getvalue(),
|
||||
'total_calls': ps.total_calls if hasattr(ps, 'total_calls') else 0,
|
||||
'timestamp': time.time()
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def get_cprofile_stats(cls) -> List[Dict[str, Any]]:
|
||||
"""Get cProfile statistics."""
|
||||
with cls._lock:
|
||||
return cls._cprofile_stats.copy()
|
||||
|
||||
# ========================================================================
|
||||
# Line-Level Profiling
|
||||
# ========================================================================
|
||||
|
||||
@classmethod
|
||||
def get_line_profile_data(cls) -> Dict[str, Any]:
|
||||
"""Get line-level profiling data."""
|
||||
with cls._lock:
|
||||
return cls._line_profile_data.copy()
|
||||
|
||||
@classmethod
|
||||
def set_line_profile_data(cls, func_name: str, data: Any) -> None:
|
||||
"""Store line-level profiling data."""
|
||||
if not cls.is_enabled():
|
||||
return
|
||||
with cls._lock:
|
||||
cls._line_profile_data[func_name] = data
|
||||
|
||||
# ========================================================================
|
||||
# Flamegraph
|
||||
# ========================================================================
|
||||
|
||||
@classmethod
|
||||
def get_flamegraph_data(cls) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Generate flamegraph-compatible data from flow records.
|
||||
|
||||
Returns list of {name, value, children} for flamegraph visualization.
|
||||
"""
|
||||
with cls._lock:
|
||||
# Convert flow records to flamegraph format
|
||||
data = []
|
||||
for record in cls._flow:
|
||||
data.append({
|
||||
'name': record.name,
|
||||
'value': record.duration_ms,
|
||||
'file': record.file,
|
||||
'line': record.line
|
||||
})
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
def export_flamegraph(cls, filepath: str) -> None:
|
||||
"""
|
||||
Export flamegraph to SVG file.
|
||||
|
||||
Note: Requires flamegraph data. For full flamegraph support,
|
||||
use py-spy: py-spy record -o profile.svg -- python script.py
|
||||
"""
|
||||
data = cls.get_flamegraph_data()
|
||||
|
||||
# Generate simple SVG flamegraph
|
||||
svg_content = cls._generate_simple_flamegraph_svg(data)
|
||||
|
||||
with open(filepath, 'w') as f:
|
||||
f.write(svg_content)
|
||||
|
||||
@classmethod
|
||||
def _generate_simple_flamegraph_svg(cls, data: List[Dict[str, Any]]) -> str:
|
||||
"""Generate a simple SVG flamegraph."""
|
||||
if not data:
|
||||
return '<svg xmlns="http://www.w3.org/2000/svg" width="800" height="100"><text x="10" y="50">No profiling data</text></svg>'
|
||||
|
||||
total_time = sum(d['value'] for d in data)
|
||||
if total_time == 0:
|
||||
total_time = 1
|
||||
|
||||
width = 800
|
||||
height = max(100, len(data) * 25 + 50)
|
||||
|
||||
svg_parts = [
|
||||
f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}">',
|
||||
'<style>rect:hover { opacity: 0.8; } text { font-family: monospace; font-size: 12px; }</style>',
|
||||
f'<text x="10" y="20">PraisonAI Profiling Flamegraph (Total: {total_time:.2f}ms)</text>'
|
||||
]
|
||||
|
||||
y = 40
|
||||
for item in sorted(data, key=lambda x: x['value'], reverse=True)[:20]:
|
||||
bar_width = max(10, (item['value'] / total_time) * (width - 20))
|
||||
color = f'hsl({int(item["value"] / total_time * 120)}, 70%, 50%)'
|
||||
|
||||
svg_parts.append(
|
||||
f'<rect x="10" y="{y}" width="{bar_width}" height="20" fill="{color}" />'
|
||||
)
|
||||
svg_parts.append(
|
||||
f'<text x="15" y="{y + 15}">{item["name"]}: {item["value"]:.2f}ms</text>'
|
||||
)
|
||||
y += 25
|
||||
|
||||
svg_parts.append('</svg>')
|
||||
return '\n'.join(svg_parts)
|
||||
|
||||
# ========================================================================
|
||||
# Export Functions
|
||||
# ========================================================================
|
||||
|
||||
@classmethod
|
||||
def export_json(cls) -> str:
|
||||
"""Export profiling data as JSON."""
|
||||
# Get summary and stats first (they acquire their own locks)
|
||||
summary = cls.get_summary()
|
||||
stats = cls.get_statistics()
|
||||
|
||||
with cls._lock:
|
||||
data = {
|
||||
'summary': summary,
|
||||
'statistics': stats,
|
||||
'timings': [asdict(t) for t in cls._timings],
|
||||
'api_calls': [asdict(a) for a in cls._api_calls],
|
||||
'streaming': [asdict(s) for s in cls._streaming],
|
||||
'memory': [asdict(m) for m in cls._memory],
|
||||
'imports': [asdict(i) for i in cls._imports],
|
||||
'flow': [asdict(f) for f in cls._flow]
|
||||
}
|
||||
return json.dumps(data, indent=2, default=str)
|
||||
|
||||
@classmethod
|
||||
def export_html(cls) -> str:
|
||||
"""Export profiling data as HTML report."""
|
||||
summary = cls.get_summary()
|
||||
stats = cls.get_statistics()
|
||||
|
||||
html = f'''<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>PraisonAI Profiling Report</title>
|
||||
<style>
|
||||
body {{ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; margin: 20px; }}
|
||||
h1 {{ color: #333; }}
|
||||
h2 {{ color: #666; border-bottom: 1px solid #ddd; padding-bottom: 5px; }}
|
||||
table {{ border-collapse: collapse; width: 100%; margin: 10px 0; }}
|
||||
th, td {{ border: 1px solid #ddd; padding: 8px; text-align: left; }}
|
||||
th {{ background-color: #f5f5f5; }}
|
||||
.metric {{ display: inline-block; margin: 10px; padding: 15px; background: #f9f9f9; border-radius: 5px; }}
|
||||
.metric-value {{ font-size: 24px; font-weight: bold; color: #0066cc; }}
|
||||
.metric-label {{ font-size: 12px; color: #666; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>PraisonAI Profiling Report</h1>
|
||||
|
||||
<h2>Summary</h2>
|
||||
<div class="metric">
|
||||
<div class="metric-value">{summary['total_time_ms']:.2f}ms</div>
|
||||
<div class="metric-label">Total Time</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="metric-value">{summary['timing_count']}</div>
|
||||
<div class="metric-label">Operations</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="metric-value">{len(cls._api_calls)}</div>
|
||||
<div class="metric-label">API Calls</div>
|
||||
</div>
|
||||
<div class="metric">
|
||||
<div class="metric-value">{len(cls._streaming)}</div>
|
||||
<div class="metric-label">Streams</div>
|
||||
</div>
|
||||
|
||||
<h2>Statistics</h2>
|
||||
<table>
|
||||
<tr><th>Metric</th><th>Value</th></tr>
|
||||
<tr><td>P50 (Median)</td><td>{stats['p50']:.2f}ms</td></tr>
|
||||
<tr><td>P95</td><td>{stats['p95']:.2f}ms</td></tr>
|
||||
<tr><td>P99</td><td>{stats['p99']:.2f}ms</td></tr>
|
||||
<tr><td>Mean</td><td>{stats['mean']:.2f}ms</td></tr>
|
||||
<tr><td>Std Dev</td><td>{stats['std_dev']:.2f}ms</td></tr>
|
||||
<tr><td>Min</td><td>{stats['min']:.2f}ms</td></tr>
|
||||
<tr><td>Max</td><td>{stats['max']:.2f}ms</td></tr>
|
||||
</table>
|
||||
|
||||
<h2>Slowest Operations</h2>
|
||||
<table>
|
||||
<tr><th>Operation</th><th>Duration (ms)</th></tr>
|
||||
{''.join(f"<tr><td>{name}</td><td>{dur:.2f}</td></tr>" for name, dur in summary['slowest_operations'])}
|
||||
</table>
|
||||
|
||||
<h2>API Calls</h2>
|
||||
<table>
|
||||
<tr><th>Endpoint</th><th>Method</th><th>Duration (ms)</th><th>Status</th></tr>
|
||||
{''.join(f"<tr><td>{a.endpoint}</td><td>{a.method}</td><td>{a.duration_ms:.2f}</td><td>{a.status_code}</td></tr>" for a in cls._api_calls[:20])}
|
||||
</table>
|
||||
|
||||
<h2>Streaming</h2>
|
||||
<table>
|
||||
<tr><th>Name</th><th>TTFT (ms)</th><th>Total (ms)</th><th>Chunks</th><th>Tokens</th></tr>
|
||||
{''.join(f"<tr><td>{s.name}</td><td>{s.ttft_ms:.2f}</td><td>{s.total_ms:.2f}</td><td>{s.chunk_count}</td><td>{s.total_tokens}</td></tr>" for s in cls._streaming[:20])}
|
||||
</table>
|
||||
</body>
|
||||
</html>'''
|
||||
return html
|
||||
|
||||
@classmethod
|
||||
def export_to_file(cls, filepath: str, format: str = "json") -> None:
|
||||
"""Export profiling data to file."""
|
||||
if format == "json":
|
||||
content = cls.export_json()
|
||||
elif format == "html":
|
||||
content = cls.export_html()
|
||||
else:
|
||||
raise ValueError(f"Unknown format: {format}")
|
||||
|
||||
with open(filepath, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
@@ -422,6 +1003,161 @@ def profile_imports():
|
||||
return ImportProfiler()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# API Profiling Decorators
|
||||
# ============================================================================
|
||||
|
||||
def profile_api(func: Optional[Callable] = None, *, endpoint: str = ""):
|
||||
"""
|
||||
Decorator to profile a function as an API call.
|
||||
|
||||
Usage:
|
||||
@profile_api(endpoint="openai/chat")
|
||||
def call_openai():
|
||||
pass
|
||||
"""
|
||||
def decorator(fn: Callable) -> Callable:
|
||||
@functools.wraps(fn)
|
||||
def wrapper(*args, **kwargs):
|
||||
if not Profiler.is_enabled():
|
||||
return fn(*args, **kwargs)
|
||||
|
||||
ep = endpoint or fn.__name__
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
finally:
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
Profiler.record_api_call(ep, "CALL", duration_ms)
|
||||
|
||||
return wrapper
|
||||
|
||||
if func is not None:
|
||||
return decorator(func)
|
||||
return decorator
|
||||
|
||||
|
||||
def profile_api_async(func: Optional[Callable] = None, *, endpoint: str = ""):
|
||||
"""
|
||||
Decorator to profile an async function as an API call.
|
||||
"""
|
||||
def decorator(fn: Callable) -> Callable:
|
||||
@functools.wraps(fn)
|
||||
async def wrapper(*args, **kwargs):
|
||||
if not Profiler.is_enabled():
|
||||
return await fn(*args, **kwargs)
|
||||
|
||||
ep = endpoint or fn.__name__
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
return await fn(*args, **kwargs)
|
||||
finally:
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
Profiler.record_api_call(ep, "CALL", duration_ms)
|
||||
|
||||
return wrapper
|
||||
|
||||
if func is not None:
|
||||
return decorator(func)
|
||||
return decorator
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# cProfile Decorator
|
||||
# ============================================================================
|
||||
|
||||
def profile_detailed(func: Optional[Callable] = None):
|
||||
"""
|
||||
Decorator for detailed cProfile profiling.
|
||||
|
||||
Usage:
|
||||
@profile_detailed
|
||||
def heavy_computation():
|
||||
pass
|
||||
"""
|
||||
def decorator(fn: Callable) -> Callable:
|
||||
@functools.wraps(fn)
|
||||
def wrapper(*args, **kwargs):
|
||||
if not Profiler.is_enabled():
|
||||
return fn(*args, **kwargs)
|
||||
|
||||
pr = cProfile.Profile()
|
||||
pr.enable()
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
finally:
|
||||
pr.disable()
|
||||
s = io.StringIO()
|
||||
ps = pstats.Stats(pr, stream=s).sort_stats('cumulative')
|
||||
ps.print_stats(20)
|
||||
Profiler._cprofile_stats.append({
|
||||
'name': fn.__name__,
|
||||
'stats': s.getvalue(),
|
||||
'timestamp': time.time()
|
||||
})
|
||||
|
||||
return wrapper
|
||||
|
||||
if func is not None:
|
||||
return decorator(func)
|
||||
return decorator
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Line-Level Profiling Decorator
|
||||
# ============================================================================
|
||||
|
||||
def profile_lines(func: Optional[Callable] = None):
|
||||
"""
|
||||
Decorator for line-level profiling.
|
||||
|
||||
Note: Requires line_profiler package for full functionality.
|
||||
Falls back to basic timing if not available.
|
||||
|
||||
Usage:
|
||||
@profile_lines
|
||||
def my_function():
|
||||
pass
|
||||
"""
|
||||
def decorator(fn: Callable) -> Callable:
|
||||
@functools.wraps(fn)
|
||||
def wrapper(*args, **kwargs):
|
||||
if not Profiler.is_enabled():
|
||||
return fn(*args, **kwargs)
|
||||
|
||||
# Try to use line_profiler if available
|
||||
try:
|
||||
from line_profiler import LineProfiler
|
||||
lp = LineProfiler()
|
||||
lp.add_function(fn)
|
||||
lp.enable()
|
||||
try:
|
||||
result = fn(*args, **kwargs)
|
||||
finally:
|
||||
lp.disable()
|
||||
s = io.StringIO()
|
||||
lp.print_stats(stream=s)
|
||||
Profiler.set_line_profile_data(fn.__name__, s.getvalue())
|
||||
return result
|
||||
except ImportError:
|
||||
# Fallback to basic timing
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
finally:
|
||||
duration_ms = (time.perf_counter() - start) * 1000
|
||||
Profiler.set_line_profile_data(fn.__name__, {
|
||||
'note': 'line_profiler not installed',
|
||||
'total_ms': duration_ms
|
||||
})
|
||||
|
||||
return wrapper
|
||||
|
||||
if func is not None:
|
||||
return decorator(func)
|
||||
return decorator
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Quick Profiling Functions
|
||||
# ============================================================================
|
||||
@@ -452,14 +1188,27 @@ def check_module_available(module_name: str) -> bool:
|
||||
# ============================================================================
|
||||
|
||||
__all__ = [
|
||||
# Core
|
||||
'Profiler',
|
||||
'StreamingTracker',
|
||||
# Decorators
|
||||
'profile',
|
||||
'profile_async',
|
||||
'profile_api',
|
||||
'profile_api_async',
|
||||
'profile_detailed',
|
||||
'profile_lines',
|
||||
# Import profiling
|
||||
'profile_imports',
|
||||
'ImportProfiler',
|
||||
# Utilities
|
||||
'time_import',
|
||||
'check_module_available',
|
||||
# Data classes
|
||||
'TimingRecord',
|
||||
'ImportRecord',
|
||||
'FlowRecord',
|
||||
'APICallRecord',
|
||||
'StreamingRecord',
|
||||
'MemoryRecord',
|
||||
]
|
||||
@@ -1 +1 @@
|
||||
__version__ = "2.3.74"
|
||||
__version__ = "2.3.75"
|
||||
@@ -0,0 +1,555 @@
|
||||
"""
|
||||
Unit tests for PraisonAI Advanced Profiler features.
|
||||
|
||||
TDD: Tests written first for new profiling capabilities:
|
||||
- API/HTTP call profiling (wall-clock time)
|
||||
- Streaming profiling (TTFT, total time)
|
||||
- Line-level profiling
|
||||
- Memory profiling
|
||||
- Flamegraph generation
|
||||
- cProfile integration
|
||||
- Statistics (p50, p95, p99)
|
||||
"""
|
||||
|
||||
import time
|
||||
import asyncio
|
||||
import pytest
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# API Call Profiling Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestAPICallProfiler:
|
||||
"""Test API/HTTP call profiling with wall-clock time."""
|
||||
|
||||
def test_api_call_record(self):
|
||||
"""Should record API call timing."""
|
||||
from praisonai.profiler import Profiler, APICallRecord
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
Profiler.record_api_call(
|
||||
endpoint="https://api.openai.com/v1/chat/completions",
|
||||
method="POST",
|
||||
duration_ms=1500.0,
|
||||
status_code=200,
|
||||
request_size=1024,
|
||||
response_size=2048
|
||||
)
|
||||
|
||||
calls = Profiler.get_api_calls()
|
||||
assert len(calls) >= 1
|
||||
assert calls[-1].endpoint == "https://api.openai.com/v1/chat/completions"
|
||||
assert calls[-1].duration_ms == 1500.0
|
||||
assert calls[-1].status_code == 200
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_api_call_context_manager(self):
|
||||
"""Should profile API call with context manager."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
with Profiler.api_call("https://api.example.com/test", method="GET") as call:
|
||||
time.sleep(0.01) # Simulate API latency
|
||||
|
||||
calls = Profiler.get_api_calls()
|
||||
assert len(calls) >= 1
|
||||
assert calls[-1].duration_ms >= 10
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_api_call_decorator(self):
|
||||
"""Should profile function as API call."""
|
||||
from praisonai.profiler import Profiler, profile_api
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
@profile_api(endpoint="test_endpoint")
|
||||
def mock_api_call():
|
||||
time.sleep(0.01)
|
||||
return {"status": "ok"}
|
||||
|
||||
result = mock_api_call()
|
||||
assert result == {"status": "ok"}
|
||||
|
||||
calls = Profiler.get_api_calls()
|
||||
assert len(calls) >= 1
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Streaming Profiling Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestStreamingProfiler:
|
||||
"""Test streaming profiling (TTFT, chunk times, total time)."""
|
||||
|
||||
def test_streaming_record(self):
|
||||
"""Should record streaming metrics."""
|
||||
from praisonai.profiler import Profiler, StreamingRecord
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
Profiler.record_streaming(
|
||||
name="chat_completion",
|
||||
ttft_ms=150.0, # Time to first token
|
||||
total_ms=2000.0,
|
||||
chunk_count=50,
|
||||
total_tokens=500
|
||||
)
|
||||
|
||||
streams = Profiler.get_streaming_records()
|
||||
assert len(streams) >= 1
|
||||
assert streams[-1].ttft_ms == 150.0
|
||||
assert streams[-1].total_ms == 2000.0
|
||||
assert streams[-1].chunk_count == 50
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_streaming_tracker(self):
|
||||
"""Should track streaming with context manager."""
|
||||
from praisonai.profiler import Profiler, StreamingTracker
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
tracker = StreamingTracker("test_stream")
|
||||
tracker.start()
|
||||
time.sleep(0.01)
|
||||
tracker.first_token() # Mark TTFT
|
||||
time.sleep(0.02)
|
||||
tracker.chunk() # Record chunk
|
||||
tracker.chunk()
|
||||
tracker.end(total_tokens=100)
|
||||
|
||||
streams = Profiler.get_streaming_records()
|
||||
assert len(streams) >= 1
|
||||
assert streams[-1].ttft_ms >= 10
|
||||
assert streams[-1].chunk_count == 2
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_streaming_context_manager(self):
|
||||
"""Should profile streaming with context manager."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
with Profiler.streaming("test_stream") as tracker:
|
||||
time.sleep(0.01)
|
||||
tracker.first_token()
|
||||
for _ in range(3):
|
||||
tracker.chunk()
|
||||
time.sleep(0.005)
|
||||
|
||||
streams = Profiler.get_streaming_records()
|
||||
assert len(streams) >= 1
|
||||
assert streams[-1].chunk_count == 3
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Memory Profiling Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestMemoryProfiler:
|
||||
"""Test memory profiling capabilities."""
|
||||
|
||||
def test_memory_record(self):
|
||||
"""Should record memory usage."""
|
||||
from praisonai.profiler import Profiler, MemoryRecord
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
Profiler.record_memory(
|
||||
name="agent_init",
|
||||
current_kb=1024.0,
|
||||
peak_kb=2048.0
|
||||
)
|
||||
|
||||
memories = Profiler.get_memory_records()
|
||||
assert len(memories) >= 1
|
||||
assert memories[-1].current_kb == 1024.0
|
||||
assert memories[-1].peak_kb == 2048.0
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_memory_context_manager(self):
|
||||
"""Should profile memory with context manager."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
with Profiler.memory("test_operation"):
|
||||
# Allocate some memory
|
||||
data = [i for i in range(10000)]
|
||||
|
||||
memories = Profiler.get_memory_records()
|
||||
assert len(memories) >= 1
|
||||
assert memories[-1].peak_kb > 0
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_memory_snapshot(self):
|
||||
"""Should take memory snapshot."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
|
||||
snapshot = Profiler.memory_snapshot()
|
||||
assert 'current_kb' in snapshot
|
||||
assert 'peak_kb' in snapshot
|
||||
|
||||
Profiler.disable()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Statistics Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestProfilerStatistics:
|
||||
"""Test statistical analysis of profiling data."""
|
||||
|
||||
def test_percentiles(self):
|
||||
"""Should calculate percentiles (p50, p95, p99)."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
# Add multiple timing records
|
||||
for i in range(100):
|
||||
Profiler.record_timing(f"op_{i}", float(i), "test")
|
||||
|
||||
stats = Profiler.get_statistics(category="test")
|
||||
|
||||
assert 'p50' in stats
|
||||
assert 'p95' in stats
|
||||
assert 'p99' in stats
|
||||
assert 'mean' in stats
|
||||
assert 'std_dev' in stats
|
||||
assert 'min' in stats
|
||||
assert 'max' in stats
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_statistics_by_category(self):
|
||||
"""Should calculate statistics per category."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
for i in range(10):
|
||||
Profiler.record_timing(f"api_{i}", float(i * 100), "api")
|
||||
Profiler.record_timing(f"func_{i}", float(i * 10), "function")
|
||||
|
||||
api_stats = Profiler.get_statistics(category="api")
|
||||
func_stats = Profiler.get_statistics(category="function")
|
||||
|
||||
assert api_stats['mean'] > func_stats['mean']
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# cProfile Integration Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestCProfileIntegration:
|
||||
"""Test cProfile integration for detailed function profiling."""
|
||||
|
||||
def test_cprofile_context_manager(self):
|
||||
"""Should profile with cProfile."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
|
||||
with Profiler.cprofile("test_operation") as stats:
|
||||
# Do some work
|
||||
result = sum(i * i for i in range(1000))
|
||||
|
||||
assert stats is not None
|
||||
assert hasattr(stats, 'total_calls') or hasattr(stats, 'stats')
|
||||
|
||||
Profiler.disable()
|
||||
|
||||
def test_cprofile_decorator(self):
|
||||
"""Should profile function with cProfile decorator."""
|
||||
from praisonai.profiler import profile_detailed
|
||||
|
||||
@profile_detailed
|
||||
def compute_heavy():
|
||||
return sum(i * i for i in range(10000))
|
||||
|
||||
result = compute_heavy()
|
||||
assert result > 0
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Flamegraph Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestFlamegraph:
|
||||
"""Test flamegraph generation."""
|
||||
|
||||
def test_flamegraph_data_generation(self):
|
||||
"""Should generate flamegraph-compatible data."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
# Record some nested operations
|
||||
with Profiler.block("outer"):
|
||||
time.sleep(0.01)
|
||||
with Profiler.block("inner"):
|
||||
time.sleep(0.01)
|
||||
|
||||
flamegraph_data = Profiler.get_flamegraph_data()
|
||||
|
||||
assert flamegraph_data is not None
|
||||
assert isinstance(flamegraph_data, (str, list, dict))
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_flamegraph_export(self):
|
||||
"""Should export flamegraph to file."""
|
||||
import tempfile
|
||||
import os
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
with Profiler.block("test_op"):
|
||||
time.sleep(0.01)
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.svg', delete=False) as f:
|
||||
filepath = f.name
|
||||
|
||||
try:
|
||||
Profiler.export_flamegraph(filepath)
|
||||
assert os.path.exists(filepath)
|
||||
finally:
|
||||
if os.path.exists(filepath):
|
||||
os.unlink(filepath)
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Line-Level Profiling Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestLineProfiler:
|
||||
"""Test line-level profiling integration."""
|
||||
|
||||
def test_line_profile_decorator(self):
|
||||
"""Should profile function line by line."""
|
||||
from praisonai.profiler import profile_lines, Profiler
|
||||
|
||||
Profiler.enable()
|
||||
|
||||
@profile_lines
|
||||
def sample_function():
|
||||
a = 1
|
||||
b = 2
|
||||
c = a + b
|
||||
return c
|
||||
|
||||
result = sample_function()
|
||||
assert result == 3
|
||||
|
||||
# Line profiling data should be available
|
||||
line_data = Profiler.get_line_profile_data()
|
||||
# May be empty if line_profiler not installed, but shouldn't error
|
||||
assert line_data is not None
|
||||
|
||||
Profiler.disable()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Report Export Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestReportExport:
|
||||
"""Test report export in various formats."""
|
||||
|
||||
def test_json_export(self):
|
||||
"""Should export report as JSON."""
|
||||
import json
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
Profiler.record_timing("test_op", 100.0, "test")
|
||||
|
||||
json_report = Profiler.export_json()
|
||||
data = json.loads(json_report)
|
||||
|
||||
assert 'timings' in data or 'summary' in data
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_html_export(self):
|
||||
"""Should export report as HTML."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
Profiler.record_timing("test_op", 100.0, "test")
|
||||
|
||||
html_report = Profiler.export_html()
|
||||
|
||||
assert '<html>' in html_report or '<!DOCTYPE' in html_report.lower()
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
def test_file_export(self):
|
||||
"""Should export report to file."""
|
||||
import tempfile
|
||||
import os
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
Profiler.record_timing("test_op", 100.0, "test")
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
|
||||
filepath = f.name
|
||||
|
||||
try:
|
||||
Profiler.export_to_file(filepath, format="json")
|
||||
assert os.path.exists(filepath)
|
||||
with open(filepath) as f:
|
||||
content = f.read()
|
||||
assert len(content) > 0
|
||||
finally:
|
||||
if os.path.exists(filepath):
|
||||
os.unlink(filepath)
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Zero Performance Impact Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestZeroPerformanceImpact:
|
||||
"""Test that profiling has zero impact when disabled."""
|
||||
|
||||
def test_disabled_no_overhead(self):
|
||||
"""Should have minimal overhead when disabled."""
|
||||
from praisonai.profiler import Profiler, profile
|
||||
|
||||
Profiler.disable()
|
||||
|
||||
@profile
|
||||
def fast_function():
|
||||
return 1 + 1
|
||||
|
||||
# Measure overhead
|
||||
iterations = 10000
|
||||
start = time.perf_counter()
|
||||
for _ in range(iterations):
|
||||
fast_function()
|
||||
duration = time.perf_counter() - start
|
||||
|
||||
# Should complete very fast (< 1ms per call on average)
|
||||
avg_us = (duration / iterations) * 1_000_000
|
||||
assert avg_us < 100 # Less than 100 microseconds per call
|
||||
|
||||
def test_no_data_when_disabled(self):
|
||||
"""Should not record data when disabled."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
Profiler.record_timing("test", 100.0, "test")
|
||||
Profiler.record_api_call("endpoint", "GET", 100.0)
|
||||
|
||||
assert len(Profiler.get_timings()) == 0
|
||||
assert len(Profiler.get_api_calls()) == 0
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Async Profiling Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestAsyncProfiling:
|
||||
"""Test async function profiling."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_api_call(self):
|
||||
"""Should profile async API calls."""
|
||||
from praisonai.profiler import Profiler, profile_api_async
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
@profile_api_async(endpoint="async_test")
|
||||
async def async_api_call():
|
||||
await asyncio.sleep(0.01)
|
||||
return {"status": "ok"}
|
||||
|
||||
result = await async_api_call()
|
||||
assert result == {"status": "ok"}
|
||||
|
||||
calls = Profiler.get_api_calls()
|
||||
assert len(calls) >= 1
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_streaming(self):
|
||||
"""Should profile async streaming."""
|
||||
from praisonai.profiler import Profiler
|
||||
|
||||
Profiler.enable()
|
||||
Profiler.clear()
|
||||
|
||||
async with Profiler.streaming_async("async_stream") as tracker:
|
||||
await asyncio.sleep(0.01)
|
||||
tracker.first_token()
|
||||
for _ in range(3):
|
||||
tracker.chunk()
|
||||
await asyncio.sleep(0.005)
|
||||
|
||||
streams = Profiler.get_streaming_records()
|
||||
assert len(streams) >= 1
|
||||
|
||||
Profiler.disable()
|
||||
Profiler.clear()
|
||||
Reference in new issue
Block a user