ifixkart-backend/swarm/benchmarks/benchmark_suite.py

103 lines
3.8 KiB
Python

"""
@benchmark BenchmarkSuite (swarm/benchmarks/benchmark_suite.py)
@purpose Empirical performance profiler measuring concurrency, latency, token consumption,
and USD execution costs across simulation scenarios.
"""
import sys, os
from dotenv import load_dotenv
load_dotenv('/mnt/fam/SERVER/IfixKartEcommerce/Backend/.env')
sys.path.insert(0, '/mnt/fam/SERVER/IfixKartEcommerce/Backend')
sys.path.insert(0, '/mnt/fam/SERVER/IfixKartEcommerce')
import time
import asyncio
import psutil
import json
from typing import List, Dict, Any
from swarm.scenarios import get_scenario, ScenarioResult
from swarm.config import swarm_settings
async def run_single_benchmark(scenario_type: str, params: Dict[str, Any]) -> Dict[str, Any]:
scenario = get_scenario(scenario_type)
t0 = time.perf_counter()
res: ScenarioResult = await scenario.run(params)
t1 = time.perf_counter()
return {
"scenario_type": scenario_type,
"wall_time_ms": int((t1 - t0) * 1000),
"execution_time_ms": res.execution_time_ms,
"prompt_tokens": res.prompt_tokens,
"completion_tokens": res.completion_tokens,
"total_tokens": res.prompt_tokens + res.completion_tokens,
"total_cost_usd": res.total_cost_usd,
"calibrated_projection": res.projection.calibrated_projected_value,
"status": "PASSED"
}
async def run_concurrency_benchmark(concurrency_level: int) -> Dict[str, Any]:
process = psutil.Process()
mem_before_mb = process.memory_info().rss / (1024 * 1024)
t0 = time.perf_counter()
tasks = []
scenarios = ["pricing_elasticity", "whatsapp_preflight", "workshop_surge", "fleet_quote"]
for i in range(concurrency_level):
sc_type = scenarios[i % len(scenarios)]
tasks.append(run_single_benchmark(sc_type, {}))
results = await asyncio.gather(*tasks)
t1 = time.perf_counter()
mem_after_mb = process.memory_info().rss / (1024 * 1024)
total_wall_ms = int((t1 - t0) * 1000)
total_tokens = sum(r["total_tokens"] for r in results)
total_cost = sum(r["total_cost_usd"] for r in results)
avg_latency = total_wall_ms / max(1, concurrency_level)
return {
"concurrency_level": concurrency_level,
"completed_jobs": len(results),
"total_wall_time_ms": total_wall_ms,
"avg_job_latency_ms": round(avg_latency, 1),
"total_tokens_consumed": total_tokens,
"total_cost_usd": round(total_cost, 6),
"memory_delta_mb": round(mem_after_mb - mem_before_mb, 2),
"throughput_jobs_per_sec": round(concurrency_level / max(0.001, (t1 - t0)), 2),
"job_details": results
}
async def execute_full_benchmark_suite() -> Dict[str, Any]:
print("=" * 70)
print("iFixKart Swarm Engine — Empirical Performance & Scalability Benchmark")
print("=" * 70)
benchmarks = []
for level in [1, 2, 4]:
print(f"Running concurrency benchmark level: {level} parallel simulations...")
bench = await run_concurrency_benchmark(level)
print(f" -> Concurrency {level}: {bench['completed_jobs']} jobs completed in {bench['total_wall_time_ms']} ms "
f"(Avg: {bench['avg_job_latency_ms']} ms/job, RAM Delta: {bench['memory_delta_mb']} MB, Tokens: {bench['total_tokens_consumed']})")
benchmarks.append(bench)
output = {
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
"model": swarm_settings.DEFAULT_PROVIDER,
"benchmarks": benchmarks
}
# Save artifact
output_path = "/mnt/fam/SERVER/IfixKartEcommerce/reports/test-results/swarm_benchmark_results.json"
os.makedirs(os.path.dirname(output_path), exist_ok=True)
with open(output_path, "w") as f:
json.dump(output, f, indent=2)
print(f"\nEmpirical benchmark results persisted to: {output_path}")
return output
if __name__ == "__main__":
asyncio.run(execute_full_benchmark_suite())