198 lines
7 KiB
Python
198 lines
7 KiB
Python
"""
|
|
@queue JobQueue (swarm/queue/job_queue.py)
|
|
@purpose Durable state machine, atomic job claiming, concurrency semaphore,
|
|
timeout enforcement, and cancellation handling.
|
|
"""
|
|
import asyncio
|
|
import os
|
|
import uuid
|
|
import time
|
|
from datetime import datetime
|
|
from typing import Optional, Dict, Any
|
|
from sqlalchemy import select, update, text
|
|
from sqlalchemy.orm import Session
|
|
from app.core.database.db_session import SessionLocal
|
|
from swarm.config import swarm_settings
|
|
from swarm.models.simulation_job import SimulationJob
|
|
from swarm.scenarios import get_scenario, ScenarioResult
|
|
|
|
# Global worker concurrency semaphore (default 2)
|
|
_SIMULATION_SEMAPHORE = asyncio.Semaphore(swarm_settings.MAX_CONCURRENT_SIMULATIONS)
|
|
_ACTIVE_CANCEL_TOKENS: Dict[str, bool] = {}
|
|
|
|
|
|
def request_job_cancellation(job_id: str) -> bool:
|
|
"""Signals an active job to cancel immediately."""
|
|
_ACTIVE_CANCEL_TOKENS[job_id] = True
|
|
session = SessionLocal()
|
|
try:
|
|
stmt = (
|
|
update(SimulationJob)
|
|
.where(SimulationJob.id == job_id)
|
|
.values(cancellation_requested=True, status="CANCELLED", completed_at=datetime.utcnow())
|
|
)
|
|
res = session.execute(stmt)
|
|
session.commit()
|
|
return res.rowcount > 0
|
|
finally:
|
|
session.close()
|
|
|
|
|
|
def is_cancellation_requested(job_id: str) -> bool:
|
|
if _ACTIVE_CANCEL_TOKENS.get(job_id, False):
|
|
return True
|
|
session = SessionLocal()
|
|
try:
|
|
job = session.query(SimulationJob).filter(SimulationJob.id == job_id).first()
|
|
return bool(job and job.cancellation_requested)
|
|
finally:
|
|
session.close()
|
|
|
|
|
|
def update_job_heartbeat(job_id: str):
|
|
"""Updates the heartbeat timestamp for an active job."""
|
|
session = SessionLocal()
|
|
try:
|
|
session.execute(
|
|
update(SimulationJob)
|
|
.where(SimulationJob.id == job_id)
|
|
.values(last_heartbeat_at=datetime.utcnow())
|
|
)
|
|
session.commit()
|
|
except Exception:
|
|
pass
|
|
finally:
|
|
session.close()
|
|
|
|
|
|
async def execute_simulation_job(job_id: str):
|
|
"""
|
|
Asynchronous runner for a single claimed job.
|
|
Enforces atomic status transitions, concurrency slots, timeouts, and error handling.
|
|
"""
|
|
worker_id = f"worker-{os.getpid()}-{uuid.uuid4().hex[:6]}"
|
|
|
|
async with _SIMULATION_SEMAPHORE:
|
|
session = SessionLocal()
|
|
job: Optional[SimulationJob] = None
|
|
try:
|
|
# 1. Atomic job claiming with row lock / verify state
|
|
job = (
|
|
session.query(SimulationJob)
|
|
.filter(SimulationJob.id == job_id, SimulationJob.status.in_(["QUEUED", "CLAIMED"]))
|
|
.with_for_update()
|
|
.first()
|
|
)
|
|
if not job or job.cancellation_requested:
|
|
return
|
|
|
|
job.status = "RUNNING"
|
|
job.worker_id = worker_id
|
|
job.started_at = datetime.utcnow()
|
|
job.last_heartbeat_at = datetime.utcnow()
|
|
session.commit()
|
|
session.refresh(job)
|
|
|
|
scenario_type = job.scenario_type
|
|
input_params = job.input_params or {}
|
|
store_id = job.store_id
|
|
session.close() # Close session during async simulation to free connection
|
|
|
|
# 2. Instantiate and run scenario with cancellation & heartbeat callbacks
|
|
scenario = get_scenario(scenario_type)
|
|
|
|
def check_cancel() -> bool:
|
|
return is_cancellation_requested(job_id)
|
|
|
|
def beat():
|
|
update_job_heartbeat(job_id)
|
|
|
|
# 3. Enforce hard execution timeout
|
|
result: ScenarioResult = await asyncio.wait_for(
|
|
scenario.run(
|
|
input_params=input_params,
|
|
store_id=store_id,
|
|
cancellation_check=check_cancel,
|
|
heartbeat_callback=beat
|
|
),
|
|
timeout=float(swarm_settings.JOB_TIMEOUT_SECONDS)
|
|
)
|
|
|
|
# 4. Success Completion Transition
|
|
session = SessionLocal()
|
|
session.execute(
|
|
update(SimulationJob)
|
|
.where(SimulationJob.id == job_id)
|
|
.values(
|
|
status="COMPLETED",
|
|
historical_baseline=result.baseline.model_dump(),
|
|
simulation_dispersion=result.dispersion.model_dump(),
|
|
calibrated_projection=result.projection.model_dump(),
|
|
report_markdown=result.report_markdown,
|
|
agent_logs=result.agent_logs,
|
|
summary_metrics=result.summary_metrics,
|
|
execution_time_ms=result.execution_time_ms,
|
|
prompt_tokens=result.prompt_tokens,
|
|
completion_tokens=result.completion_tokens,
|
|
total_cost_usd=result.total_cost_usd,
|
|
completed_at=datetime.utcnow(),
|
|
last_heartbeat_at=datetime.utcnow()
|
|
)
|
|
)
|
|
session.commit()
|
|
|
|
except asyncio.TimeoutError:
|
|
session = SessionLocal()
|
|
session.execute(
|
|
update(SimulationJob)
|
|
.where(SimulationJob.id == job_id)
|
|
.values(
|
|
status="TIMED_OUT",
|
|
error_message=f"Simulation exceeded hard timeout of {swarm_settings.JOB_TIMEOUT_SECONDS}s.",
|
|
completed_at=datetime.utcnow()
|
|
)
|
|
)
|
|
session.commit()
|
|
|
|
except InterruptedError as ie:
|
|
session = SessionLocal()
|
|
session.execute(
|
|
update(SimulationJob)
|
|
.where(SimulationJob.id == job_id)
|
|
.values(
|
|
status="CANCELLED",
|
|
error_message=str(ie),
|
|
completed_at=datetime.utcnow()
|
|
)
|
|
)
|
|
session.commit()
|
|
|
|
except Exception as e:
|
|
session = SessionLocal()
|
|
job = session.query(SimulationJob).filter(SimulationJob.id == job_id).first()
|
|
if job and job.retry_count < job.max_retries and "rate" in str(e).lower():
|
|
# Eligible for transient retry
|
|
session.execute(
|
|
update(SimulationJob)
|
|
.where(SimulationJob.id == job_id)
|
|
.values(
|
|
status="QUEUED",
|
|
retry_count=job.retry_count + 1,
|
|
worker_id=None,
|
|
error_message=f"Transient failure (Retry {job.retry_count + 1}): {str(e)}"
|
|
)
|
|
)
|
|
else:
|
|
session.execute(
|
|
update(SimulationJob)
|
|
.where(SimulationJob.id == job_id)
|
|
.values(
|
|
status="FAILED",
|
|
error_message=str(e),
|
|
completed_at=datetime.utcnow()
|
|
)
|
|
)
|
|
session.commit()
|
|
finally:
|
|
_ACTIVE_CANCEL_TOKENS.pop(job_id, None)
|
|
session.close()
|