103 lines
3.8 KiB
Python
103 lines
3.8 KiB
Python
"""
|
|
@benchmark BenchmarkSuite (swarm/benchmarks/benchmark_suite.py)
|
|
@purpose Empirical performance profiler measuring concurrency, latency, token consumption,
|
|
and USD execution costs across simulation scenarios.
|
|
"""
|
|
import sys, os
|
|
from dotenv import load_dotenv
|
|
load_dotenv('/mnt/fam/SERVER/IfixKartEcommerce/Backend/.env')
|
|
sys.path.insert(0, '/mnt/fam/SERVER/IfixKartEcommerce/Backend')
|
|
sys.path.insert(0, '/mnt/fam/SERVER/IfixKartEcommerce')
|
|
|
|
import time
|
|
import asyncio
|
|
import psutil
|
|
import json
|
|
from typing import List, Dict, Any
|
|
from swarm.scenarios import get_scenario, ScenarioResult
|
|
from swarm.config import swarm_settings
|
|
|
|
|
|
async def run_single_benchmark(scenario_type: str, params: Dict[str, Any]) -> Dict[str, Any]:
|
|
scenario = get_scenario(scenario_type)
|
|
t0 = time.perf_counter()
|
|
res: ScenarioResult = await scenario.run(params)
|
|
t1 = time.perf_counter()
|
|
|
|
return {
|
|
"scenario_type": scenario_type,
|
|
"wall_time_ms": int((t1 - t0) * 1000),
|
|
"execution_time_ms": res.execution_time_ms,
|
|
"prompt_tokens": res.prompt_tokens,
|
|
"completion_tokens": res.completion_tokens,
|
|
"total_tokens": res.prompt_tokens + res.completion_tokens,
|
|
"total_cost_usd": res.total_cost_usd,
|
|
"calibrated_projection": res.projection.calibrated_projected_value,
|
|
"status": "PASSED"
|
|
}
|
|
|
|
|
|
async def run_concurrency_benchmark(concurrency_level: int) -> Dict[str, Any]:
|
|
process = psutil.Process()
|
|
mem_before_mb = process.memory_info().rss / (1024 * 1024)
|
|
t0 = time.perf_counter()
|
|
|
|
tasks = []
|
|
scenarios = ["pricing_elasticity", "whatsapp_preflight", "workshop_surge", "fleet_quote"]
|
|
for i in range(concurrency_level):
|
|
sc_type = scenarios[i % len(scenarios)]
|
|
tasks.append(run_single_benchmark(sc_type, {}))
|
|
|
|
results = await asyncio.gather(*tasks)
|
|
t1 = time.perf_counter()
|
|
mem_after_mb = process.memory_info().rss / (1024 * 1024)
|
|
|
|
total_wall_ms = int((t1 - t0) * 1000)
|
|
total_tokens = sum(r["total_tokens"] for r in results)
|
|
total_cost = sum(r["total_cost_usd"] for r in results)
|
|
avg_latency = total_wall_ms / max(1, concurrency_level)
|
|
|
|
return {
|
|
"concurrency_level": concurrency_level,
|
|
"completed_jobs": len(results),
|
|
"total_wall_time_ms": total_wall_ms,
|
|
"avg_job_latency_ms": round(avg_latency, 1),
|
|
"total_tokens_consumed": total_tokens,
|
|
"total_cost_usd": round(total_cost, 6),
|
|
"memory_delta_mb": round(mem_after_mb - mem_before_mb, 2),
|
|
"throughput_jobs_per_sec": round(concurrency_level / max(0.001, (t1 - t0)), 2),
|
|
"job_details": results
|
|
}
|
|
|
|
|
|
async def execute_full_benchmark_suite() -> Dict[str, Any]:
|
|
print("=" * 70)
|
|
print("iFixKart Swarm Engine — Empirical Performance & Scalability Benchmark")
|
|
print("=" * 70)
|
|
|
|
benchmarks = []
|
|
for level in [1, 2, 4]:
|
|
print(f"Running concurrency benchmark level: {level} parallel simulations...")
|
|
bench = await run_concurrency_benchmark(level)
|
|
print(f" -> Concurrency {level}: {bench['completed_jobs']} jobs completed in {bench['total_wall_time_ms']} ms "
|
|
f"(Avg: {bench['avg_job_latency_ms']} ms/job, RAM Delta: {bench['memory_delta_mb']} MB, Tokens: {bench['total_tokens_consumed']})")
|
|
benchmarks.append(bench)
|
|
|
|
output = {
|
|
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"model": swarm_settings.DEFAULT_PROVIDER,
|
|
"benchmarks": benchmarks
|
|
}
|
|
|
|
# Save artifact
|
|
output_path = "/mnt/fam/SERVER/IfixKartEcommerce/reports/test-results/swarm_benchmark_results.json"
|
|
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
|
with open(output_path, "w") as f:
|
|
json.dump(output, f, indent=2)
|
|
print(f"\nEmpirical benchmark results persisted to: {output_path}")
|
|
|
|
return output
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(execute_full_benchmark_suite())
|