236 lines
10 KiB
Python
236 lines
10 KiB
Python
"""
|
|
@scenario WhatsAppPreflightScenario (swarm/scenarios/whatsapp_preflight.py)
|
|
@purpose Simulates customer reception, click-through rates, and phone number block/spam risk
|
|
for proposed WhatsApp campaigns before broadcasting via OpenWA.
|
|
"""
|
|
import asyncio
|
|
import time
|
|
import json
|
|
from typing import Dict, Any, Callable, Optional, List
|
|
from pydantic import BaseModel, Field
|
|
from swarm.core.persona_generator import generate_personas, AgentPersona
|
|
from swarm.core.calibration import (
|
|
fetch_historical_baseline,
|
|
compute_simulation_dispersion,
|
|
synthesize_calibrated_projection,
|
|
)
|
|
from swarm.core.synthesizer import generate_markdown_report
|
|
from swarm.providers import get_llm_provider
|
|
from swarm.scenarios.base_scenario import BaseScenario, ScenarioResult
|
|
|
|
|
|
class WhatsAppArchetypeReasoning(BaseModel):
|
|
archetype_id: str
|
|
spam_reaction: str
|
|
interview_quote: str
|
|
|
|
|
|
class WhatsAppPreflightScenario(BaseScenario):
|
|
async def run(
|
|
self,
|
|
input_params: Dict[str, Any],
|
|
store_id: int = 1,
|
|
cancellation_check: Optional[Callable[[], bool]] = None,
|
|
heartbeat_callback: Optional[Callable[[], None]] = None
|
|
) -> ScenarioResult:
|
|
start_time = time.perf_counter()
|
|
|
|
# 1. Parse inputs
|
|
message_template = str(input_params.get("message_template", "Hey there! Exclusive 20% off all display repairs this weekend at iFixKart!"))
|
|
offer_type = str(input_params.get("offer_type", "20% Discount"))
|
|
frequency = int(input_params.get("frequency_per_month", 2))
|
|
target_segment = str(input_params.get("target_segment", "Past Customers"))
|
|
|
|
# 2. Baseline
|
|
baseline = fetch_historical_baseline("whatsapp_preflight", store_id)
|
|
if heartbeat_callback:
|
|
heartbeat_callback()
|
|
|
|
if cancellation_check and cancellation_check():
|
|
raise InterruptedError("Simulation cancelled by user.")
|
|
|
|
# 3. Personas
|
|
personas = generate_personas(count=50, seed=101)
|
|
agent_outcomes: List[float] = []
|
|
agent_logs: List[Dict[str, Any]] = []
|
|
|
|
spam_block_count = 0
|
|
click_count = 0
|
|
|
|
# Frequency penalty
|
|
freq_irritation_multiplier = 1.0 + max(0, frequency - 2) * 0.35
|
|
|
|
for p in personas:
|
|
# Willingness to click
|
|
click_prob = min(0.65, max(0.02, baseline.empirical_mean + (p.price_sensitivity * 0.12) - (p.whatsapp_irritation_threshold * 0.05)))
|
|
|
|
# Risk of reporting spam / blocking
|
|
block_hazard = p.whatsapp_irritation_threshold * freq_irritation_multiplier * 0.15
|
|
if frequency > 4:
|
|
block_hazard += 0.20
|
|
|
|
will_block = block_hazard > 0.45
|
|
if will_block:
|
|
spam_block_count += 1
|
|
click_prob = 0.0
|
|
elif click_prob > 0.15:
|
|
click_count += 1
|
|
|
|
agent_outcomes.append(click_prob)
|
|
agent_logs.append({
|
|
"agent_id": p.agent_id,
|
|
"archetype": p.archetype_name,
|
|
"click_prob": round(click_prob, 3),
|
|
"block_hazard": round(block_hazard, 3),
|
|
"decision": "BLOCK" if will_block else ("CLICK" if click_prob > 0.15 else "IGNORE")
|
|
})
|
|
|
|
if heartbeat_callback:
|
|
heartbeat_callback()
|
|
|
|
# 4. Dispersion & Calibration
|
|
dispersion = compute_simulation_dispersion(agent_outcomes)
|
|
projection = synthesize_calibrated_projection(baseline, dispersion)
|
|
|
|
# 5. LLM Synthesis
|
|
provider = get_llm_provider()
|
|
archetype_breakdown = []
|
|
interview_samples = []
|
|
total_prompt_tokens = 0
|
|
total_completion_tokens = 0
|
|
total_cost = 0.0
|
|
|
|
# Group by archetype
|
|
arch_groups = {}
|
|
for p in personas:
|
|
arch_groups.setdefault(p.archetype_id, []).append(p)
|
|
|
|
for arch_id, p_list in arch_groups.items():
|
|
if cancellation_check and cancellation_check():
|
|
raise InterruptedError("Simulation cancelled by user.")
|
|
|
|
async def _eval_wa_archetype(arch_id: str, p_list: List[AgentPersona]):
|
|
sub_logs = [log for log in agent_logs if log["agent_id"] in {p.agent_id for p in p_list}]
|
|
sub_clicks = sum(1 for log in sub_logs if log["decision"] == "CLICK")
|
|
sub_blocks = sum(1 for log in sub_logs if log["decision"] == "BLOCK")
|
|
|
|
prompt = (
|
|
f"You are simulating customer reactions to an iFixKart WhatsApp broadcast message.\n"
|
|
f"Message Copy: \"{message_template}\"\n"
|
|
f"Monthly frequency: {frequency} messages/month.\n"
|
|
f"Archetype: {p_list[0].archetype_name} (Total: {len(p_list)}, Clicks: {sub_clicks}, Blocks: {sub_blocks}).\n"
|
|
f"Provide a 1-sentence reaction and a simulated customer quote about this broadcast."
|
|
)
|
|
try:
|
|
reasoning, usage = await provider.generate_structured(
|
|
prompt=prompt,
|
|
schema=WhatsAppArchetypeReasoning,
|
|
system_prompt="You are a messaging deliverability & customer sentiment analyst."
|
|
)
|
|
return {
|
|
"breakdown": {
|
|
"name": p_list[0].archetype_name,
|
|
"share": f"{len(p_list)/len(personas):.0%}",
|
|
"rate": f"{sub_clicks/max(1, len(p_list)):.1%}",
|
|
"friction": reasoning.spam_reaction
|
|
},
|
|
"sample": {
|
|
"persona": f"{p_list[0].archetype_name} ({p_list[0].name})",
|
|
"quote": reasoning.interview_quote
|
|
},
|
|
"usage": usage
|
|
}
|
|
except Exception:
|
|
return {
|
|
"breakdown": {
|
|
"name": p_list[0].archetype_name,
|
|
"share": f"{len(p_list)/len(personas):.0%}",
|
|
"rate": f"{sub_clicks/max(1, len(p_list)):.1%}",
|
|
"friction": "Tolerates targeted repair offers if infrequent."
|
|
},
|
|
"sample": {
|
|
"persona": f"{p_list[0].archetype_name} ({p_list[0].name})",
|
|
"quote": "I don't mind a useful discount message once a month, but if you text weekly I'll block the number."
|
|
},
|
|
"usage": None
|
|
}
|
|
|
|
results = await asyncio.gather(*(_eval_wa_archetype(a_id, p_l) for a_id, p_l in arch_groups.items()))
|
|
for r in results:
|
|
archetype_breakdown.append(r["breakdown"])
|
|
interview_samples.append(r["sample"])
|
|
if r["usage"]:
|
|
total_prompt_tokens += r["usage"].prompt_tokens
|
|
total_completion_tokens += r["usage"].completion_tokens
|
|
total_cost += r["usage"].estimated_cost_usd
|
|
|
|
# 6. Block Risk Score (0 - 100)
|
|
simulated_block_rate = spam_block_count / max(1, len(personas))
|
|
block_risk_score = min(100.0, simulated_block_rate * 300.0)
|
|
|
|
if block_risk_score < 25.0:
|
|
risk_verdict = "LOW RISK (Safe for broadcast)"
|
|
elif block_risk_score < 50.0:
|
|
risk_verdict = "MODERATE RISK (Monitor unsubscribes)"
|
|
else:
|
|
risk_verdict = "HIGH RISK (Threat of WhatsApp number restriction)"
|
|
|
|
summary_metrics = {
|
|
"message_length_chars": len(message_template),
|
|
"monthly_frequency": frequency,
|
|
"target_segment": target_segment,
|
|
"baseline_ctr_pct": round(baseline.empirical_mean * 100.0, 2),
|
|
"projected_ctr_pct": round(projection.calibrated_projected_value * 100.0, 2),
|
|
"baseline_open_rate_pct": round(baseline.empirical_mean * 100.0 * 2.5, 2),
|
|
"projected_open_rate_pct": round(projection.calibrated_projected_value * 100.0 * 2.5, 2),
|
|
"simulated_block_rate_pct": round(simulated_block_rate * 100.0, 2),
|
|
"whatsapp_block_risk_score": round(block_risk_score, 1),
|
|
"spam_block_risk_score": round(block_risk_score, 1),
|
|
"risk_verdict": risk_verdict,
|
|
"safety_verdict": risk_verdict
|
|
}
|
|
|
|
# 7. Render Report
|
|
elapsed_ms = int((time.perf_counter() - start_time) * 1000)
|
|
telemetry = {
|
|
"formatted_inputs": json.dumps(input_params, indent=2),
|
|
"recommendation_summary": (
|
|
f"Block risk score is {block_risk_score:.1f}/100. "
|
|
f"{'Message copy is acceptable for broadcasting.' if block_risk_score < 30.0 else 'High opt-out risk! Reduce frequency from ' + str(frequency) + ' to 2/month and add clear opt-out instructions.'}"
|
|
),
|
|
"execution_time_ms": elapsed_ms,
|
|
"model_name": getattr(provider, "model", "mock"),
|
|
"total_tokens": total_prompt_tokens + total_completion_tokens,
|
|
"prompt_tokens": total_prompt_tokens,
|
|
"completion_tokens": total_completion_tokens,
|
|
"total_cost_usd": round(total_cost, 6)
|
|
}
|
|
|
|
report_md = generate_markdown_report(
|
|
scenario_type="whatsapp_preflight",
|
|
scenario_title=f"WhatsApp Campaign Pre-Flight (Risk Score: {block_risk_score:.1f}/100)",
|
|
input_params=input_params,
|
|
baseline=baseline,
|
|
dispersion=dispersion,
|
|
projection=projection,
|
|
archetype_breakdown=archetype_breakdown,
|
|
interview_samples=interview_samples,
|
|
telemetry=telemetry
|
|
)
|
|
|
|
return ScenarioResult(
|
|
scenario_type="whatsapp_preflight",
|
|
scenario_title=f"WhatsApp Pre-Flight ({risk_verdict})",
|
|
baseline=baseline,
|
|
dispersion=dispersion,
|
|
projection=projection,
|
|
summary_metrics=summary_metrics,
|
|
archetype_breakdown=archetype_breakdown,
|
|
interview_samples=interview_samples,
|
|
agent_logs=agent_logs,
|
|
report_markdown=report_md,
|
|
execution_time_ms=elapsed_ms,
|
|
prompt_tokens=total_prompt_tokens,
|
|
completion_tokens=total_completion_tokens,
|
|
total_cost_usd=round(total_cost, 6)
|
|
)
|