""" @queue JobQueue (swarm/queue/job_queue.py) @purpose Durable state machine, atomic job claiming, concurrency semaphore, timeout enforcement, and cancellation handling. """ import asyncio import os import uuid import time from datetime import datetime from typing import Optional, Dict, Any from sqlalchemy import select, update, text from sqlalchemy.orm import Session from app.core.database.db_session import SessionLocal from swarm.config import swarm_settings from swarm.models.simulation_job import SimulationJob from swarm.scenarios import get_scenario, ScenarioResult # Global worker concurrency semaphore (default 2) _SIMULATION_SEMAPHORE = asyncio.Semaphore(swarm_settings.MAX_CONCURRENT_SIMULATIONS) _ACTIVE_CANCEL_TOKENS: Dict[str, bool] = {} def request_job_cancellation(job_id: str) -> bool: """Signals an active job to cancel immediately.""" _ACTIVE_CANCEL_TOKENS[job_id] = True session = SessionLocal() try: stmt = ( update(SimulationJob) .where(SimulationJob.id == job_id) .values(cancellation_requested=True, status="CANCELLED", completed_at=datetime.utcnow()) ) res = session.execute(stmt) session.commit() return res.rowcount > 0 finally: session.close() def is_cancellation_requested(job_id: str) -> bool: if _ACTIVE_CANCEL_TOKENS.get(job_id, False): return True session = SessionLocal() try: job = session.query(SimulationJob).filter(SimulationJob.id == job_id).first() return bool(job and job.cancellation_requested) finally: session.close() def update_job_heartbeat(job_id: str): """Updates the heartbeat timestamp for an active job.""" session = SessionLocal() try: session.execute( update(SimulationJob) .where(SimulationJob.id == job_id) .values(last_heartbeat_at=datetime.utcnow()) ) session.commit() except Exception: pass finally: session.close() async def execute_simulation_job(job_id: str): """ Asynchronous runner for a single claimed job. Enforces atomic status transitions, concurrency slots, timeouts, and error handling. """ worker_id = f"worker-{os.getpid()}-{uuid.uuid4().hex[:6]}" async with _SIMULATION_SEMAPHORE: session = SessionLocal() job: Optional[SimulationJob] = None try: # 1. Atomic job claiming with row lock / verify state job = ( session.query(SimulationJob) .filter(SimulationJob.id == job_id, SimulationJob.status.in_(["QUEUED", "CLAIMED"])) .with_for_update() .first() ) if not job or job.cancellation_requested: return job.status = "RUNNING" job.worker_id = worker_id job.started_at = datetime.utcnow() job.last_heartbeat_at = datetime.utcnow() session.commit() session.refresh(job) scenario_type = job.scenario_type input_params = job.input_params or {} store_id = job.store_id session.close() # Close session during async simulation to free connection # 2. Instantiate and run scenario with cancellation & heartbeat callbacks scenario = get_scenario(scenario_type) def check_cancel() -> bool: return is_cancellation_requested(job_id) def beat(): update_job_heartbeat(job_id) # 3. Enforce hard execution timeout result: ScenarioResult = await asyncio.wait_for( scenario.run( input_params=input_params, store_id=store_id, cancellation_check=check_cancel, heartbeat_callback=beat ), timeout=float(swarm_settings.JOB_TIMEOUT_SECONDS) ) # 4. Success Completion Transition session = SessionLocal() session.execute( update(SimulationJob) .where(SimulationJob.id == job_id) .values( status="COMPLETED", historical_baseline=result.baseline.model_dump(), simulation_dispersion=result.dispersion.model_dump(), calibrated_projection=result.projection.model_dump(), report_markdown=result.report_markdown, agent_logs=result.agent_logs, summary_metrics=result.summary_metrics, execution_time_ms=result.execution_time_ms, prompt_tokens=result.prompt_tokens, completion_tokens=result.completion_tokens, total_cost_usd=result.total_cost_usd, completed_at=datetime.utcnow(), last_heartbeat_at=datetime.utcnow() ) ) session.commit() except asyncio.TimeoutError: session = SessionLocal() session.execute( update(SimulationJob) .where(SimulationJob.id == job_id) .values( status="TIMED_OUT", error_message=f"Simulation exceeded hard timeout of {swarm_settings.JOB_TIMEOUT_SECONDS}s.", completed_at=datetime.utcnow() ) ) session.commit() except InterruptedError as ie: session = SessionLocal() session.execute( update(SimulationJob) .where(SimulationJob.id == job_id) .values( status="CANCELLED", error_message=str(ie), completed_at=datetime.utcnow() ) ) session.commit() except Exception as e: session = SessionLocal() job = session.query(SimulationJob).filter(SimulationJob.id == job_id).first() if job and job.retry_count < job.max_retries and "rate" in str(e).lower(): # Eligible for transient retry session.execute( update(SimulationJob) .where(SimulationJob.id == job_id) .values( status="QUEUED", retry_count=job.retry_count + 1, worker_id=None, error_message=f"Transient failure (Retry {job.retry_count + 1}): {str(e)}" ) ) else: session.execute( update(SimulationJob) .where(SimulationJob.id == job_id) .values( status="FAILED", error_message=str(e), completed_at=datetime.utcnow() ) ) session.commit() finally: _ACTIVE_CANCEL_TOKENS.pop(job_id, None) session.close()