201 lines
7.2 KiB
Python
201 lines
7.2 KiB
Python
"""
|
|
Async fleet orchestrator: run N customer agents with bounded concurrency,
|
|
realistic staggered arrival and per-endpoint telemetry.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import math
|
|
import random
|
|
import time
|
|
from collections import Counter, defaultdict
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from swarm.agents.customer import CustomerAgent, JourneyResult
|
|
from swarm.agents.personas import Persona, persona_for_agent
|
|
|
|
|
|
def _percentile(values: list[float], pct: float) -> float:
|
|
if not values:
|
|
return 0.0
|
|
ordered = sorted(values)
|
|
idx = min(len(ordered) - 1, max(0, int(math.ceil(pct / 100.0 * len(ordered))) - 1))
|
|
return ordered[idx]
|
|
|
|
|
|
class Metrics:
|
|
def __init__(self) -> None:
|
|
self.lock = asyncio.Lock()
|
|
self.requests: Counter[tuple[str, int]] = Counter()
|
|
self.latencies: dict[str, list[float]] = defaultdict(list)
|
|
self.request_errors: Counter[str] = Counter()
|
|
self.outcomes: Counter[str] = Counter()
|
|
self.orders: list[dict[str, Any]] = []
|
|
self.errors: Counter[str] = Counter()
|
|
self.completed = 0
|
|
|
|
async def record_request(self, path: str, status: int, seconds: float, error: str = "") -> None:
|
|
# Normalise dynamic IDs so latency buckets stay stable.
|
|
key = path.split("?")[0]
|
|
if key.startswith("/api/v1/orders/"):
|
|
key = "/api/v1/orders/{id}"
|
|
if key.startswith("/api/v1/catalog/products/detail/"):
|
|
key = "/api/v1/catalog/products/detail/{slug}"
|
|
async with self.lock:
|
|
self.requests[(key, status)] += 1
|
|
self.latencies[key].append(seconds)
|
|
if error:
|
|
self.request_errors[error] += 1
|
|
|
|
async def record_result(self, result: JourneyResult) -> None:
|
|
async with self.lock:
|
|
self.completed += 1
|
|
self.outcomes[result.outcome] += 1
|
|
if result.outcome == "ORDER_CONFIRMED":
|
|
self.orders.append(
|
|
{
|
|
"order_id": result.order_id,
|
|
"order_no": result.order_no,
|
|
"amount": result.amount,
|
|
"email": result.email,
|
|
"persona_id": result.persona_id,
|
|
}
|
|
)
|
|
elif result.outcome == "ERROR":
|
|
message = result.error[:180]
|
|
self.errors[message] += 1
|
|
|
|
def summary(self, started_at: datetime, duration_s: float, count: int) -> dict[str, Any]:
|
|
total_requests = sum(self.requests.values())
|
|
statuses: Counter[int] = Counter()
|
|
for (_path, status), n in self.requests.items():
|
|
statuses[status] += n
|
|
latency = {}
|
|
for path, values in self.latencies.items():
|
|
latency[path] = {
|
|
"count": len(values),
|
|
"p50_ms": round(_percentile(values, 50) * 1000, 1),
|
|
"p95_ms": round(_percentile(values, 95) * 1000, 1),
|
|
"p99_ms": round(_percentile(values, 99) * 1000, 1),
|
|
"max_ms": round(max(values) * 1000, 1),
|
|
}
|
|
revenue = round(sum(o["amount"] for o in self.orders), 2)
|
|
return {
|
|
"started_at": started_at.isoformat(),
|
|
"duration_seconds": round(duration_s, 1),
|
|
"agents_requested": count,
|
|
"agents_completed": self.completed,
|
|
"outcomes": dict(self.outcomes),
|
|
"orders_confirmed": len(self.orders),
|
|
"revenue": revenue,
|
|
"aov": round(revenue / len(self.orders), 2) if self.orders else 0.0,
|
|
"total_http_requests": total_requests,
|
|
"http_statuses": {str(k): v for k, v in sorted(statuses.items())},
|
|
"request_errors": dict(self.request_errors),
|
|
"endpoint_latency": latency,
|
|
"top_errors": [{"error": k, "count": v} for k, v in self.errors.most_common(15)],
|
|
"orders_sample": self.orders[:20],
|
|
}
|
|
|
|
|
|
async def run_fleet(
|
|
*,
|
|
base_url: str,
|
|
accounts: list[dict[str, Any]],
|
|
base_personas: list[Persona],
|
|
count: int,
|
|
concurrency: int,
|
|
think_time_scale: float,
|
|
ramp_seconds: float,
|
|
seed: int,
|
|
request_timeout: float,
|
|
progress_every: int = 100,
|
|
) -> dict[str, Any]:
|
|
count = min(count, len(accounts))
|
|
if count <= 0:
|
|
raise ValueError("no agent accounts supplied")
|
|
if not base_personas:
|
|
raise ValueError("empty persona bank")
|
|
|
|
metrics = Metrics()
|
|
rng = random.Random(seed)
|
|
plans = [
|
|
(accounts[i], persona_for_agent(base_personas, i, rng))
|
|
for i in range(count)
|
|
]
|
|
|
|
limits = httpx.Limits(
|
|
max_connections=max(50, concurrency * 2),
|
|
max_keepalive_connections=max(20, concurrency),
|
|
keepalive_expiry=30.0,
|
|
)
|
|
semaphore = asyncio.Semaphore(concurrency)
|
|
started_wall = datetime.now(timezone.utc)
|
|
started = time.perf_counter()
|
|
|
|
async with httpx.AsyncClient(
|
|
base_url=base_url,
|
|
timeout=request_timeout,
|
|
limits=limits,
|
|
follow_redirects=False,
|
|
) as client:
|
|
|
|
async def one(index: int, account: dict[str, Any], persona: Persona) -> JourneyResult:
|
|
if ramp_seconds > 0 and count > 1:
|
|
await asyncio.sleep(ramp_seconds * index / (count - 1))
|
|
agent = CustomerAgent(
|
|
client=client,
|
|
agent_index=index,
|
|
account=account,
|
|
persona=persona,
|
|
rng=random.Random(seed + index * 7919),
|
|
think_time_scale=think_time_scale,
|
|
request_timeout=request_timeout,
|
|
metrics=metrics,
|
|
)
|
|
async with semaphore:
|
|
result = await agent.run()
|
|
await metrics.record_result(result)
|
|
async with metrics.lock:
|
|
done = metrics.completed
|
|
if done % max(1, progress_every) == 0 or done == count:
|
|
elapsed = time.perf_counter() - started
|
|
rate = done / elapsed if elapsed else 0
|
|
print(
|
|
f"[fleet] {done}/{count} agents | {rate:.1f}/s | "
|
|
f"orders={len(metrics.orders)} | outcomes={dict(metrics.outcomes)}",
|
|
flush=True,
|
|
)
|
|
return result
|
|
|
|
tasks = [
|
|
asyncio.create_task(one(i, account, persona))
|
|
for i, (account, persona) in enumerate(plans)
|
|
]
|
|
results = await asyncio.gather(*tasks, return_exceptions=True)
|
|
|
|
for result in results:
|
|
if isinstance(result, Exception):
|
|
await metrics.record_result(
|
|
JourneyResult(
|
|
agent_index=-1,
|
|
email="",
|
|
persona_id="",
|
|
outcome="ERROR",
|
|
error=f"orchestrator: {type(result).__name__}: {result}",
|
|
)
|
|
)
|
|
|
|
duration = time.perf_counter() - started
|
|
return metrics.summary(started_wall, duration, count)
|
|
|
|
|
|
def write_report(report: dict[str, Any], output: Path) -> None:
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
output.write_text(json.dumps(report, indent=2))
|
|
print(f"[fleet] report written to {output}")
|