ifixkart-backend/swarm/queue/job_queue.py

198 lines
7 KiB
Python

"""
@queue JobQueue (swarm/queue/job_queue.py)
@purpose Durable state machine, atomic job claiming, concurrency semaphore,
timeout enforcement, and cancellation handling.
"""
import asyncio
import os
import uuid
import time
from datetime import datetime
from typing import Optional, Dict, Any
from sqlalchemy import select, update, text
from sqlalchemy.orm import Session
from app.core.database.db_session import SessionLocal
from swarm.config import swarm_settings
from swarm.models.simulation_job import SimulationJob
from swarm.scenarios import get_scenario, ScenarioResult
# Global worker concurrency semaphore (default 2)
_SIMULATION_SEMAPHORE = asyncio.Semaphore(swarm_settings.MAX_CONCURRENT_SIMULATIONS)
_ACTIVE_CANCEL_TOKENS: Dict[str, bool] = {}
def request_job_cancellation(job_id: str) -> bool:
"""Signals an active job to cancel immediately."""
_ACTIVE_CANCEL_TOKENS[job_id] = True
session = SessionLocal()
try:
stmt = (
update(SimulationJob)
.where(SimulationJob.id == job_id)
.values(cancellation_requested=True, status="CANCELLED", completed_at=datetime.utcnow())
)
res = session.execute(stmt)
session.commit()
return res.rowcount > 0
finally:
session.close()
def is_cancellation_requested(job_id: str) -> bool:
if _ACTIVE_CANCEL_TOKENS.get(job_id, False):
return True
session = SessionLocal()
try:
job = session.query(SimulationJob).filter(SimulationJob.id == job_id).first()
return bool(job and job.cancellation_requested)
finally:
session.close()
def update_job_heartbeat(job_id: str):
"""Updates the heartbeat timestamp for an active job."""
session = SessionLocal()
try:
session.execute(
update(SimulationJob)
.where(SimulationJob.id == job_id)
.values(last_heartbeat_at=datetime.utcnow())
)
session.commit()
except Exception:
pass
finally:
session.close()
async def execute_simulation_job(job_id: str):
"""
Asynchronous runner for a single claimed job.
Enforces atomic status transitions, concurrency slots, timeouts, and error handling.
"""
worker_id = f"worker-{os.getpid()}-{uuid.uuid4().hex[:6]}"
async with _SIMULATION_SEMAPHORE:
session = SessionLocal()
job: Optional[SimulationJob] = None
try:
# 1. Atomic job claiming with row lock / verify state
job = (
session.query(SimulationJob)
.filter(SimulationJob.id == job_id, SimulationJob.status.in_(["QUEUED", "CLAIMED"]))
.with_for_update()
.first()
)
if not job or job.cancellation_requested:
return
job.status = "RUNNING"
job.worker_id = worker_id
job.started_at = datetime.utcnow()
job.last_heartbeat_at = datetime.utcnow()
session.commit()
session.refresh(job)
scenario_type = job.scenario_type
input_params = job.input_params or {}
store_id = job.store_id
session.close() # Close session during async simulation to free connection
# 2. Instantiate and run scenario with cancellation & heartbeat callbacks
scenario = get_scenario(scenario_type)
def check_cancel() -> bool:
return is_cancellation_requested(job_id)
def beat():
update_job_heartbeat(job_id)
# 3. Enforce hard execution timeout
result: ScenarioResult = await asyncio.wait_for(
scenario.run(
input_params=input_params,
store_id=store_id,
cancellation_check=check_cancel,
heartbeat_callback=beat
),
timeout=float(swarm_settings.JOB_TIMEOUT_SECONDS)
)
# 4. Success Completion Transition
session = SessionLocal()
session.execute(
update(SimulationJob)
.where(SimulationJob.id == job_id)
.values(
status="COMPLETED",
historical_baseline=result.baseline.model_dump(),
simulation_dispersion=result.dispersion.model_dump(),
calibrated_projection=result.projection.model_dump(),
report_markdown=result.report_markdown,
agent_logs=result.agent_logs,
summary_metrics=result.summary_metrics,
execution_time_ms=result.execution_time_ms,
prompt_tokens=result.prompt_tokens,
completion_tokens=result.completion_tokens,
total_cost_usd=result.total_cost_usd,
completed_at=datetime.utcnow(),
last_heartbeat_at=datetime.utcnow()
)
)
session.commit()
except asyncio.TimeoutError:
session = SessionLocal()
session.execute(
update(SimulationJob)
.where(SimulationJob.id == job_id)
.values(
status="TIMED_OUT",
error_message=f"Simulation exceeded hard timeout of {swarm_settings.JOB_TIMEOUT_SECONDS}s.",
completed_at=datetime.utcnow()
)
)
session.commit()
except InterruptedError as ie:
session = SessionLocal()
session.execute(
update(SimulationJob)
.where(SimulationJob.id == job_id)
.values(
status="CANCELLED",
error_message=str(ie),
completed_at=datetime.utcnow()
)
)
session.commit()
except Exception as e:
session = SessionLocal()
job = session.query(SimulationJob).filter(SimulationJob.id == job_id).first()
if job and job.retry_count < job.max_retries and "rate" in str(e).lower():
# Eligible for transient retry
session.execute(
update(SimulationJob)
.where(SimulationJob.id == job_id)
.values(
status="QUEUED",
retry_count=job.retry_count + 1,
worker_id=None,
error_message=f"Transient failure (Retry {job.retry_count + 1}): {str(e)}"
)
)
else:
session.execute(
update(SimulationJob)
.where(SimulationJob.id == job_id)
.values(
status="FAILED",
error_message=str(e),
completed_at=datetime.utcnow()
)
)
session.commit()
finally:
_ACTIVE_CANCEL_TOKENS.pop(job_id, None)
session.close()