ifixkart-backend/swarm/agents/orchestrator.py

201 lines
7.2 KiB
Python

"""
Async fleet orchestrator: run N customer agents with bounded concurrency,
realistic staggered arrival and per-endpoint telemetry.
"""
from __future__ import annotations
import asyncio
import json
import math
import random
import time
from collections import Counter, defaultdict
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import httpx
from swarm.agents.customer import CustomerAgent, JourneyResult
from swarm.agents.personas import Persona, persona_for_agent
def _percentile(values: list[float], pct: float) -> float:
if not values:
return 0.0
ordered = sorted(values)
idx = min(len(ordered) - 1, max(0, int(math.ceil(pct / 100.0 * len(ordered))) - 1))
return ordered[idx]
class Metrics:
def __init__(self) -> None:
self.lock = asyncio.Lock()
self.requests: Counter[tuple[str, int]] = Counter()
self.latencies: dict[str, list[float]] = defaultdict(list)
self.request_errors: Counter[str] = Counter()
self.outcomes: Counter[str] = Counter()
self.orders: list[dict[str, Any]] = []
self.errors: Counter[str] = Counter()
self.completed = 0
async def record_request(self, path: str, status: int, seconds: float, error: str = "") -> None:
# Normalise dynamic IDs so latency buckets stay stable.
key = path.split("?")[0]
if key.startswith("/api/v1/orders/"):
key = "/api/v1/orders/{id}"
if key.startswith("/api/v1/catalog/products/detail/"):
key = "/api/v1/catalog/products/detail/{slug}"
async with self.lock:
self.requests[(key, status)] += 1
self.latencies[key].append(seconds)
if error:
self.request_errors[error] += 1
async def record_result(self, result: JourneyResult) -> None:
async with self.lock:
self.completed += 1
self.outcomes[result.outcome] += 1
if result.outcome == "ORDER_CONFIRMED":
self.orders.append(
{
"order_id": result.order_id,
"order_no": result.order_no,
"amount": result.amount,
"email": result.email,
"persona_id": result.persona_id,
}
)
elif result.outcome == "ERROR":
message = result.error[:180]
self.errors[message] += 1
def summary(self, started_at: datetime, duration_s: float, count: int) -> dict[str, Any]:
total_requests = sum(self.requests.values())
statuses: Counter[int] = Counter()
for (_path, status), n in self.requests.items():
statuses[status] += n
latency = {}
for path, values in self.latencies.items():
latency[path] = {
"count": len(values),
"p50_ms": round(_percentile(values, 50) * 1000, 1),
"p95_ms": round(_percentile(values, 95) * 1000, 1),
"p99_ms": round(_percentile(values, 99) * 1000, 1),
"max_ms": round(max(values) * 1000, 1),
}
revenue = round(sum(o["amount"] for o in self.orders), 2)
return {
"started_at": started_at.isoformat(),
"duration_seconds": round(duration_s, 1),
"agents_requested": count,
"agents_completed": self.completed,
"outcomes": dict(self.outcomes),
"orders_confirmed": len(self.orders),
"revenue": revenue,
"aov": round(revenue / len(self.orders), 2) if self.orders else 0.0,
"total_http_requests": total_requests,
"http_statuses": {str(k): v for k, v in sorted(statuses.items())},
"request_errors": dict(self.request_errors),
"endpoint_latency": latency,
"top_errors": [{"error": k, "count": v} for k, v in self.errors.most_common(15)],
"orders_sample": self.orders[:20],
}
async def run_fleet(
*,
base_url: str,
accounts: list[dict[str, Any]],
base_personas: list[Persona],
count: int,
concurrency: int,
think_time_scale: float,
ramp_seconds: float,
seed: int,
request_timeout: float,
progress_every: int = 100,
) -> dict[str, Any]:
count = min(count, len(accounts))
if count <= 0:
raise ValueError("no agent accounts supplied")
if not base_personas:
raise ValueError("empty persona bank")
metrics = Metrics()
rng = random.Random(seed)
plans = [
(accounts[i], persona_for_agent(base_personas, i, rng))
for i in range(count)
]
limits = httpx.Limits(
max_connections=max(50, concurrency * 2),
max_keepalive_connections=max(20, concurrency),
keepalive_expiry=30.0,
)
semaphore = asyncio.Semaphore(concurrency)
started_wall = datetime.now(timezone.utc)
started = time.perf_counter()
async with httpx.AsyncClient(
base_url=base_url,
timeout=request_timeout,
limits=limits,
follow_redirects=False,
) as client:
async def one(index: int, account: dict[str, Any], persona: Persona) -> JourneyResult:
if ramp_seconds > 0 and count > 1:
await asyncio.sleep(ramp_seconds * index / (count - 1))
agent = CustomerAgent(
client=client,
agent_index=index,
account=account,
persona=persona,
rng=random.Random(seed + index * 7919),
think_time_scale=think_time_scale,
request_timeout=request_timeout,
metrics=metrics,
)
async with semaphore:
result = await agent.run()
await metrics.record_result(result)
async with metrics.lock:
done = metrics.completed
if done % max(1, progress_every) == 0 or done == count:
elapsed = time.perf_counter() - started
rate = done / elapsed if elapsed else 0
print(
f"[fleet] {done}/{count} agents | {rate:.1f}/s | "
f"orders={len(metrics.orders)} | outcomes={dict(metrics.outcomes)}",
flush=True,
)
return result
tasks = [
asyncio.create_task(one(i, account, persona))
for i, (account, persona) in enumerate(plans)
]
results = await asyncio.gather(*tasks, return_exceptions=True)
for result in results:
if isinstance(result, Exception):
await metrics.record_result(
JourneyResult(
agent_index=-1,
email="",
persona_id="",
outcome="ERROR",
error=f"orchestrator: {type(result).__name__}: {result}",
)
)
duration = time.perf_counter() - started
return metrics.summary(started_wall, duration, count)
def write_report(report: dict[str, Any], output: Path) -> None:
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(report, indent=2))
print(f"[fleet] report written to {output}")