fix(benchmarks): rotate framework order each timed round

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/praisonai-agents/benchmarks/execution_benchmark.py

@@ -5,23 +5,39 @@ PraisonAI Agents - Real Execution Benchmark
 Benchmarks ACTUAL agent execution with LLM API calls.
 Tests single agent running a simple task across frameworks.
 
+Each framework gets untimed warm-up calls first (imports, client set-up and
+the first connection are not what this measures), then the timed runs are
+interleaved round by round so network drift hits every framework alike.
+A run that raises or returns an empty answer is counted as a failure, not
+timed. The summary reports the median.
+
 Usage:
     export OPENAI_API_KEY=your_key
     python benchmarks/execution_benchmark.py
-    python benchmarks/execution_benchmark.py --no-save
-    python benchmarks/execution_benchmark.py --iterations 5
+    python benchmarks/execution_benchmark.py --save
+    python benchmarks/execution_benchmark.py --iterations 10
     python benchmarks/execution_benchmark.py --model gpt-4o --iterations 10
 """
 
-import time
-import os
 import argparse
+import os
+import statistics
+import time
 from importlib.metadata import version as get_version
 
 # Defaults (can be overridden via CLI)
 DEFAULT_MODEL = "gpt-4o-mini"
-DEFAULT_ITERATIONS = 3
+DEFAULT_ITERATIONS = 10
+DEFAULT_WARMUP = 1
 DEFAULT_PROMPT = "What is 2+2? Reply with just the number."
+INSTRUCTIONS = "You are a helpful assistant. Be very brief."
+
+METHODS = {
+    'PraisonAI': 'agent.start()',
+    'PraisonAI (LiteLLM)': 'agent.start()',
+    'Agno': 'agent.run()',
+    'CrewAI': 'crew.kickoff()',
+}
 
 
 def get_package_versions():
@@ -35,260 +51,235 @@ def get_package_versions():
     for name, pkg in packages.items():
         try:
             versions[name] = get_version(pkg)
-        except:
+        except Exception:
             versions[name] = 'not installed'
     return versions
 
 
-def benchmark_praisonai(model, iterations, prompt):
-    """Benchmark PraisonAI agent execution using agent.start()"""
+def praisonai_runner(model, prompt):
+    """PraisonAI: a new agent per run; only agent.start() is timed."""
     from praisonaiagents import Agent
-    
-    print("\n--- PraisonAI ---")
-    print("Method: agent.start()")
-    
-    times = []
-    for i in range(iterations):
-        agent = Agent(
-            name="Calculator",
-            instructions="You are a helpful assistant. Be very brief.",
-            llm=model
-        )
-        
-        start = time.perf_counter()
-        response = agent.start(prompt, output="silent")
-        elapsed = time.perf_counter() - start
-        
-        times.append(elapsed)
-        print(f"  Run {i+1}: {elapsed:.2f}s - Response: {str(response)[:50]}")
-    
-    avg = sum(times) / len(times)
-    print(f"  Average: {avg:.2f}s")
-    return avg
+
+    def build():
+        agent = Agent(name="Calculator", instructions=INSTRUCTIONS, llm=model)
+        return lambda: agent.start(prompt, output="silent")
+    return build
 
 
-def benchmark_praisonai_litellm(model, iterations, prompt):
-    """Benchmark PraisonAI with LiteLLM backend using agent.start()"""
-    from praisonaiagents import Agent
-    
-    print("\n--- PraisonAI (LiteLLM) ---")
-    print("Method: agent.start()")
-    
-    times = []
-    for i in range(iterations):
-        agent = Agent(
-            name="Calculator",
-            instructions="You are a helpful assistant. Be very brief.",
-            llm=f"openai/{model}"
-        )
-        
-        start = time.perf_counter()
-        response = agent.start(prompt, output="silent")
-        elapsed = time.perf_counter() - start
-        
-        times.append(elapsed)
-        print(f"  Run {i+1}: {elapsed:.2f}s - Response: {str(response)[:50]}")
-    
-    avg = sum(times) / len(times)
-    print(f"  Average: {avg:.2f}s")
-    return avg
-
-
-def benchmark_agno(model, iterations, prompt):
-    """Benchmark Agno agent execution using agent.run()"""
+def agno_runner(model, prompt):
+    """Agno: a new agent per run; only agent.run() is timed."""
     from agno.agent import Agent
     from agno.models.openai import OpenAIChat
-    
-    print("\n--- Agno ---")
-    print("Method: agent.run()")
-    
-    times = []
+
+    # One model object per runner: each OpenAIChat opens its own HTTP client,
+    # so a fresh one per round would put a new connection inside the timer,
+    # while PraisonAI and CrewAI agents share theirs.
+    llm = OpenAIChat(id=model)
+
+    def build():
+        agent = Agent(model=llm, instructions=[INSTRUCTIONS])
+
+        def call():
+            response = agent.run(prompt)
+            # Agno reports a failed run in ``status`` and puts the error text in
+            # ``content``, so the content alone would pass for an answer.
+            if str(getattr(response, 'status', '')).lower().endswith('error'):
+                raise RuntimeError(f"Agno run failed: {str(response.content)[:200]}")
+            return getattr(response, 'content', response)
+        return call
+    return build
+
+
+def crewai_runner(model, prompt):
+    """CrewAI: a new one-agent crew per run; only crew.kickoff() is timed."""
+    from crewai import Agent, Crew, Task
+
+    def build():
+        agent = Agent(role="Calculator", goal="Answer math questions",
+                      backstory=INSTRUCTIONS, llm=model, verbose=False)
+        task = Task(description=prompt, expected_output="A number", agent=agent)
+        return Crew(agents=[agent], tasks=[task], verbose=False).kickoff
+    return build
+
+
+def build_runners(model, prompt):
+    """Return {framework: build()} for every installed framework.
+
+    ``build()`` creates fresh objects (untimed) and returns the call to time.
+    """
+    factories = [
+        ('PraisonAI', lambda: praisonai_runner(model, prompt)),
+        ('PraisonAI (LiteLLM)', lambda: praisonai_runner(f"openai/{model}", prompt)),
+        ('Agno', lambda: agno_runner(model, prompt)),
+        ('CrewAI', lambda: crewai_runner(model, prompt)),
+    ]
+    runners = {}
+    for name, factory in factories:
+        try:
+            runners[name] = factory()
+        except ImportError as e:
+            print(f"  {name}: skipped ({e})")
+    return runners
+
+
+def answer_text(response):
+    """The answer as text, or '' when the run produced nothing usable."""
+    if response is None:
+        return ''
+    text = str(response).strip()
+    return '' if text == 'None' else text
+
+
+def run_benchmark(runners, iterations, warmup):
+    """Warm up each framework, then time interleaved rounds.
+
+    A failed warm-up is reported but the framework stays in the timed rounds,
+    so one transient error does not discard it. The order within a round
+    rotates every round. Object construction happens
+    before the timer starts; only the execution call is timed.
+
+    Returns {framework: {"times": [...], "failures": [...]}}.
+    """
+    results = {name: {"times": [], "failures": []} for name in runners}
+
+    for name, build in runners.items():
+        for _ in range(warmup):
+            try:
+                if not answer_text(build()()):
+                    raise RuntimeError("empty answer")
+            except Exception as e:
+                print(f"  {name}: warm-up failed ({type(e).__name__}: {str(e)[:120]})")
+
+    # Rotate the start each round so no framework always holds the same slot.
+    names = list(runners)
     for i in range(iterations):
-        agent = Agent(
-            model=OpenAIChat(id=model),
-            instructions=["You are a helpful assistant. Be very brief."],
-        )
-        
-        start = time.perf_counter()
-        response = agent.run(prompt)
-        elapsed = time.perf_counter() - start
-        
-        times.append(elapsed)
-        # Extract content from response
-        content = str(response.content)[:50] if hasattr(response, 'content') else str(response)[:50]
-        print(f"  Run {i+1}: {elapsed:.2f}s - Response: {content}")
-    
-    avg = sum(times) / len(times)
-    print(f"  Average: {avg:.2f}s")
-    return avg
+        shift = i % len(names) if names else 0
+        for name in names[shift:] + names[:shift]:
+            build = runners[name]
+            try:
+                call = build()
+                start = time.perf_counter()
+                text = answer_text(call())
+                elapsed = time.perf_counter() - start
+                error = None if text else "empty answer"
+            except Exception as e:
+                text, error = '', f"{type(e).__name__}: {e}"
+            if error:
+                results[name]["failures"].append(error)
+                print(f"  Round {i + 1} {name}: FAILED ({error[:120]})")
+            else:
+                results[name]["times"].append(elapsed)
+                print(f"  Round {i + 1} {name}: {elapsed:.2f}s - {text[:40]}")
+    return results
 
 
-def benchmark_crewai(model, iterations, prompt):
-    """Benchmark CrewAI agent execution using crew.kickoff()"""
-    from crewai import Agent, Task, Crew
-    
-    print("\n--- CrewAI ---")
-    print("Method: crew.kickoff()")
-    
-    times = []
-    for i in range(iterations):
-        agent = Agent(
-            role="Calculator",
-            goal="Answer math questions",
-            backstory="You are a helpful assistant.",
-            llm=model,
-            verbose=False
-        )
-        
-        task = Task(
-            description=prompt,
-            expected_output="A number",
-            agent=agent
-        )
-        
-        crew = Crew(
-            agents=[agent],
-            tasks=[task],
-            verbose=False
-        )
-        
-        start = time.perf_counter()
-        response = crew.kickoff()
-        elapsed = time.perf_counter() - start
-        
-        times.append(elapsed)
-        print(f"  Run {i+1}: {elapsed:.2f}s - Response: {str(response)[:50]}")
-    
-    avg = sum(times) / len(times)
-    print(f"  Average: {avg:.2f}s")
-    return avg
+def summarise(results):
+    """Rows of (framework, median, mean, ok, failed), fastest median first."""
+    rows = []
+    for name, r in results.items():
+        times = r["times"]
+        if times:
+            rows.append((name, statistics.median(times), statistics.fmean(times),
+                         len(times), len(r["failures"])))
+    return sorted(rows, key=lambda row: row[1])
 
 
-def save_results(results: dict, model: str, iterations: int, prompt: str):
+def save_results(rows, results, model, iterations, warmup, prompt):
     """Save results to markdown file."""
-    import os
     from datetime import datetime
-    
+
     filepath = os.path.join(os.path.dirname(__file__), 'EXECUTION_BENCHMARK_RESULTS.md')
     versions = get_package_versions()
-    
-    # Sort by time
-    sorted_results = sorted(results.items(), key=lambda x: x[1])
-    fastest = sorted_results[0][1]
-    
+    fastest = rows[0][1]
+
     with open(filepath, 'w') as f:
         f.write('# PraisonAI Agents - Real Execution Benchmark\n\n')
         f.write(f'**Generated:** {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}\n')
         f.write(f'**Model:** {model}\n')
-        f.write(f'**Iterations:** {iterations}\n')
+        f.write(f'**Iterations:** {iterations} interleaved rounds after {warmup} warm-up call(s) per framework\n')
         f.write(f'**Prompt:** "{prompt}"\n\n')
         f.write('## Results\n\n')
-        f.write('| Framework | Method | Avg Time | Relative |\n')
-        f.write('|-----------|--------|----------|----------|\n')
-        
-        methods = {
-            'PraisonAI': 'agent.start()',
-            'PraisonAI (LiteLLM)': 'agent.start()',
-            'Agno': 'agent.run()',
-            'CrewAI': 'crew.kickoff()',
-        }
-        
-        for name, avg in sorted_results:
-            ratio = avg / fastest
-            method = methods.get(name, 'unknown')
+        f.write('| Framework | Method | Median | Mean | Runs ok / failed | Relative |\n')
+        f.write('|-----------|--------|--------|------|------------------|----------|\n')
+        for name, median, mean, ok, failed in rows:
+            ratio = median / fastest
+            method = METHODS.get(name, 'unknown')
             if ratio == 1.0:
-                f.write(f'| **{name}** | `{method}` | **{avg:.2f}s** | **1.00x (fastest)** |\n')
+                f.write(f'| **{name}** | `{method}` | **{median:.2f}s** | {mean:.2f}s | {ok} / {failed} | **1.00x (fastest)** |\n')
             else:
-                f.write(f'| {name} | `{method}` | {avg:.2f}s | {ratio:.2f}x |\n')
-        
+                f.write(f'| {name} | `{method}` | {median:.2f}s | {mean:.2f}s | {ok} / {failed} | {ratio:.2f}x |\n')
+        excluded = [name for name, r in results.items() if not r["times"]]
+        if excluded:
+            f.write(f'\nNo successful runs (not ranked): {", ".join(excluded)}\n')
+
         f.write('\n## Package Versions\n\n')
         f.write('| Package | Version |\n')
         f.write('|---------|--------|\n')
         for pkg, ver in versions.items():
             f.write(f'| {pkg} | {ver} |\n')
-        
+
         f.write('\n## How to Reproduce\n\n')
         f.write('```bash\n')
         f.write('export OPENAI_API_KEY=your_key\n')
-        f.write('cd praisonai-agents\n')
-        f.write('python benchmarks/execution_benchmark.py\n')
+        f.write('cd src/praisonai-agents\n')
+        f.write('python benchmarks/execution_benchmark.py --save\n')
         f.write('```\n')
-    
+
     print(f'\nResults saved to: {filepath}')
 
 
-if __name__ == '__main__':
+def main():
     parser = argparse.ArgumentParser(description='PraisonAI Agents - Real Execution Benchmark')
-    parser.add_argument('--save', action='store_true', help='Save results to file')
+    parser.add_argument('--save', action='store_true', help='Save results to EXECUTION_BENCHMARK_RESULTS.md')
     parser.add_argument('--model', '-m', type=str, default=DEFAULT_MODEL, help=f'Model to use (default: {DEFAULT_MODEL})')
-    parser.add_argument('--iterations', '-i', type=int, default=DEFAULT_ITERATIONS, help=f'Number of iterations (default: {DEFAULT_ITERATIONS})')
+    parser.add_argument('--iterations', '-i', type=int, default=DEFAULT_ITERATIONS, help=f'Timed rounds (default: {DEFAULT_ITERATIONS})')
+    parser.add_argument('--warmup', '-w', type=int, default=DEFAULT_WARMUP, help=f'Untimed warm-up calls per framework (default: {DEFAULT_WARMUP})')
     parser.add_argument('--prompt', '-p', type=str, default=DEFAULT_PROMPT, help=f'Prompt to use (default: "{DEFAULT_PROMPT}")')
     args = parser.parse_args()
-    
-    # Check API key
+    if args.iterations < 1:
+        parser.error('--iterations must be at least 1')
+    if args.warmup < 0:
+        parser.error('--warmup must be 0 or more')
+
     if not os.environ.get('OPENAI_API_KEY'):
         print("ERROR: Set OPENAI_API_KEY environment variable")
-        exit(1)
-    
-    model = args.model
-    iterations = args.iterations
-    prompt = args.prompt
-    
+        return 1
+
     print('=' * 60)
     print('PraisonAI Agents - Real Execution Benchmark')
     print('=' * 60)
-    print(f'\nModel: {model}')
-    print(f'Iterations: {iterations}')
-    print(f'Prompt: "{prompt}"')
-    
-    versions = get_package_versions()
+    print(f'\nModel: {args.model}')
+    print(f'Rounds: {args.iterations} (after {args.warmup} warm-up call(s) per framework)')
+    print(f'Prompt: "{args.prompt}"')
     print('\nPackage versions:')
-    for pkg, ver in versions.items():
+    for pkg, ver in get_package_versions().items():
         print(f'  {pkg}: {ver}')
-    
-    results = {}
-    
-    # Run benchmarks
-    try:
-        results['PraisonAI'] = benchmark_praisonai(model, iterations, prompt)
-    except Exception as e:
-        print(f"PraisonAI error: {e}")
-    
-    try:
-        results['PraisonAI (LiteLLM)'] = benchmark_praisonai_litellm(model, iterations, prompt)
-    except Exception as e:
-        print(f"PraisonAI (LiteLLM) error: {e}")
-    
-    try:
-        results['Agno'] = benchmark_agno(model, iterations, prompt)
-    except Exception as e:
-        print(f"Agno error: {e}")
-    
-    try:
-        results['CrewAI'] = benchmark_crewai(model, iterations, prompt)
-    except Exception as e:
-        print(f"CrewAI error: {e}")
-    
-    # Summary
+    print()
+
+    runners = build_runners(args.model, args.prompt)
+    results = run_benchmark(runners, args.iterations, args.warmup)
+    rows = summarise(results)
+
     print('\n' + '=' * 60)
-    print('SUMMARY')
+    print('SUMMARY (median of successful runs)')
     print('=' * 60)
-    
-    if results:
-        sorted_results = sorted(results.items(), key=lambda x: x[1])
-        fastest = sorted_results[0][1]
-        
-        print(f"\n{'Framework':<25} {'Avg Time':<12} {'Relative':<10}")
-        print('-' * 47)
-        
-        for name, avg in sorted_results:
-            ratio = avg / fastest
-            print(f'{name:<25} {avg:<12.2f}s {ratio:.2f}x')
-    
+    if rows:
+        fastest = rows[0][1]
+        print(f"\n{'Framework':<22} {'Median':>8} {'Mean':>8} {'ok/failed':>10} {'Relative':>9}")
+        print('-' * 61)
+        for name, median, mean, ok, failed in rows:
+            print(f'{name:<22} {median:>7.2f}s {mean:>7.2f}s {ok:>5}/{failed:<4} {median / fastest:>8.2f}x')
+    for name, r in results.items():
+        if not r["times"]:
+            print(f'{name:<22} no successful runs: {(r["failures"] or ["?"])[0][:80]}')
     print('\n' + '=' * 60)
-    
-    # Save results (if --save)
-    if results and args.save:
-        save_results(results, model, iterations, prompt)
+
+    if rows and args.save:
+        save_results(rows, results, args.model, args.iterations, args.warmup, args.prompt)
     elif not args.save:
         print('\nResults not saved (use --save flag to save results to file)')
+    return 0
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

src/praisonai-agents/benchmarks/simple_benchmark.py

@@ -109,17 +109,11 @@ def run_benchmark(skip_crewai: bool = False, quiet: bool = False,
         pass
     
     try:
-        from langchain_core.tools import tool as langchain_tool
         from langchain_openai import ChatOpenAI
         from langgraph.prebuilt import create_react_agent
-        
-        @langchain_tool
-        def get_weather_lg(city: Literal['nyc', 'sf']):
-            """Get weather info."""
-            return 'sunny' if city == 'sf' else 'cloudy'
-        
+
         results['LangGraph'] = measure(
-            lambda: create_react_agent(model=ChatOpenAI(model='gpt-4o-mini'), tools=[get_weather_lg])
+            lambda: create_react_agent(model=ChatOpenAI(model='gpt-4o-mini'), tools=[])
         )
     except ImportError:
         pass
@@ -129,19 +123,12 @@ def run_benchmark(skip_crewai: bool = False, quiet: bool = False,
     else:
         try:
             from crewai.agent import Agent as CrewAgent
-            from crewai.tools import tool as crewai_tool
-
-            @crewai_tool("Weather Tool")
-            def get_weather_crew(city: Literal['nyc', 'sf']):
-                """Get weather info."""
-                return 'sunny' if city == 'sf' else 'cloudy'
 
             results['CrewAI'] = measure(
                 lambda: CrewAgent(
                     role='Weather Agent',
                     goal='Provide weather info',
                     backstory='A weather expert',
-                    tools=[get_weather_crew],
                     verbose=False
                 )
             )

src/praisonai-agents/tests/unit/test_execution_benchmark_rules.py

@@ -0,0 +1,85 @@
+"""Counting rules of benchmarks/execution_benchmark.py, with fake runners.
+
+No framework is imported and nothing leaves the machine: each runner is a
+``build()`` that returns the call to time, as the script's real runners do.
+"""
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+SCRIPT = Path(__file__).resolve().parents[2] / "benchmarks" / "execution_benchmark.py"
+
+
+@pytest.fixture(scope="module")
+def bench():
+    spec = importlib.util.spec_from_file_location("execution_benchmark", SCRIPT)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def runner(answers, log, name):
+    """A build() whose calls return ``answers`` in order (an Exception is raised)."""
+    answers = iter(answers)
+
+    def build():
+        log.append(("build", name))
+
+        def call():
+            log.append(("call", name))
+            answer = next(answers)
+            if isinstance(answer, Exception):
+                raise answer
+            return answer
+        return call
+    return build
+
+
+def test_warm_up_calls_are_not_timed_and_rounds_interleave(bench):
+    log = []
+    runners = {"A": runner(["w", "a1", "a2"], log, "A"), "B": runner(["w", "b1", "b2"], log, "B")}
+    results = bench.run_benchmark(runners, iterations=2, warmup=1)
+    assert [len(r["times"]) for r in results.values()] == [2, 2]
+    calls = [name for kind, name in log if kind == "call"]
+    assert calls == ["A", "B", "A", "B", "B", "A"]  # warm-ups, round 1, round 2 rotated
+
+
+def test_errors_and_empty_answers_are_failures_not_timings(bench):
+    log = []
+    runners = {"A": runner(["w", RuntimeError("401"), None, "None", "  ", "4"], log, "A")}
+    results = bench.run_benchmark(runners, iterations=5, warmup=1)
+    assert len(results["A"]["times"]) == 1
+    assert len(results["A"]["failures"]) == 4
+    assert "RuntimeError: 401" in results["A"]["failures"][0]
+
+
+def test_a_failed_warm_up_does_not_drop_the_framework(bench):
+    log = []
+    runners = {"A": runner([RuntimeError("timeout"), "a1", "a2"], log, "A")}
+    results = bench.run_benchmark(runners, iterations=2, warmup=1)
+    assert len(results["A"]["times"]) == 2
+
+
+def test_summary_ranks_by_median_and_skips_frameworks_without_a_success(bench):
+    results = {
+        "slow": {"times": [1.0, 1.1, 9.0], "failures": []},
+        "fast": {"times": [0.5, 0.6, 0.7], "failures": ["x"]},
+        "broken": {"times": [], "failures": ["401"]},
+    }
+    rows = bench.summarise(results)
+    assert [r[0] for r in rows] == ["fast", "slow"]
+    name, median, mean, ok, failed = rows[1]
+    assert (median, ok, failed) == (1.1, 3, 0)
+    assert mean == pytest.approx(11.1 / 3)
+
+
+def test_only_the_call_is_timed(bench, monkeypatch):
+    clock = iter([10.0, 10.5])  # start / end of the one timed call; build() reads no clock
+
+    def build():
+        return lambda: "4"
+
+    monkeypatch.setattr(bench.time, "perf_counter", lambda: next(clock))
+    results = bench.run_benchmark({"A": build}, iterations=1, warmup=0)
+    assert results["A"]["times"] == [0.5]