""" @benchmark BenchmarkSuite (swarm/benchmarks/benchmark_suite.py) @purpose Empirical performance profiler measuring concurrency, latency, token consumption, and USD execution costs across simulation scenarios. """ import sys, os from dotenv import load_dotenv load_dotenv('/mnt/fam/SERVER/IfixKartEcommerce/Backend/.env') sys.path.insert(0, '/mnt/fam/SERVER/IfixKartEcommerce/Backend') sys.path.insert(0, '/mnt/fam/SERVER/IfixKartEcommerce') import time import asyncio import psutil import json from typing import List, Dict, Any from swarm.scenarios import get_scenario, ScenarioResult from swarm.config import swarm_settings async def run_single_benchmark(scenario_type: str, params: Dict[str, Any]) -> Dict[str, Any]: scenario = get_scenario(scenario_type) t0 = time.perf_counter() res: ScenarioResult = await scenario.run(params) t1 = time.perf_counter() return { "scenario_type": scenario_type, "wall_time_ms": int((t1 - t0) * 1000), "execution_time_ms": res.execution_time_ms, "prompt_tokens": res.prompt_tokens, "completion_tokens": res.completion_tokens, "total_tokens": res.prompt_tokens + res.completion_tokens, "total_cost_usd": res.total_cost_usd, "calibrated_projection": res.projection.calibrated_projected_value, "status": "PASSED" } async def run_concurrency_benchmark(concurrency_level: int) -> Dict[str, Any]: process = psutil.Process() mem_before_mb = process.memory_info().rss / (1024 * 1024) t0 = time.perf_counter() tasks = [] scenarios = ["pricing_elasticity", "whatsapp_preflight", "workshop_surge", "fleet_quote"] for i in range(concurrency_level): sc_type = scenarios[i % len(scenarios)] tasks.append(run_single_benchmark(sc_type, {})) results = await asyncio.gather(*tasks) t1 = time.perf_counter() mem_after_mb = process.memory_info().rss / (1024 * 1024) total_wall_ms = int((t1 - t0) * 1000) total_tokens = sum(r["total_tokens"] for r in results) total_cost = sum(r["total_cost_usd"] for r in results) avg_latency = total_wall_ms / max(1, concurrency_level) return { "concurrency_level": concurrency_level, "completed_jobs": len(results), "total_wall_time_ms": total_wall_ms, "avg_job_latency_ms": round(avg_latency, 1), "total_tokens_consumed": total_tokens, "total_cost_usd": round(total_cost, 6), "memory_delta_mb": round(mem_after_mb - mem_before_mb, 2), "throughput_jobs_per_sec": round(concurrency_level / max(0.001, (t1 - t0)), 2), "job_details": results } async def execute_full_benchmark_suite() -> Dict[str, Any]: print("=" * 70) print("iFixKart Swarm Engine — Empirical Performance & Scalability Benchmark") print("=" * 70) benchmarks = [] for level in [1, 2, 4]: print(f"Running concurrency benchmark level: {level} parallel simulations...") bench = await run_concurrency_benchmark(level) print(f" -> Concurrency {level}: {bench['completed_jobs']} jobs completed in {bench['total_wall_time_ms']} ms " f"(Avg: {bench['avg_job_latency_ms']} ms/job, RAM Delta: {bench['memory_delta_mb']} MB, Tokens: {bench['total_tokens_consumed']})") benchmarks.append(bench) output = { "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ"), "model": swarm_settings.DEFAULT_PROVIDER, "benchmarks": benchmarks } # Save artifact output_path = "/mnt/fam/SERVER/IfixKartEcommerce/reports/test-results/swarm_benchmark_results.json" os.makedirs(os.path.dirname(output_path), exist_ok=True) with open(output_path, "w") as f: json.dump(output, f, indent=2) print(f"\nEmpirical benchmark results persisted to: {output_path}") return output if __name__ == "__main__": asyncio.run(execute_full_benchmark_suite())