""" @recovery Recovery (swarm/queue/recovery.py) @purpose Heartbeat monitor & startup zombie job reaper for crash recovery. """ from datetime import datetime, timedelta from sqlalchemy import text from app.core.database.db_session import engine_core from swarm.config import swarm_settings def reap_stale_jobs() -> int: """ Identifies jobs left in 'RUNNING' or 'CLAIMED' state from an earlier server crash (where last_heartbeat_at is older than ZOMBIE_REAP_MINUTES). Transitions them safely and atomically to 'FAILED_INTERRUPTED' directly on engine_core. Returns count of reaped jobs. """ cutoff = datetime.utcnow() - timedelta(minutes=swarm_settings.ZOMBIE_REAP_MINUTES) reaped_count = 0 try: stmt = text(""" UPDATE simulation_jobs SET status = 'FAILED_INTERRUPTED', error_message = 'Job interrupted by server crash or worker restart. No heartbeat detected within recovery window.', completed_at = UTC_TIMESTAMP() WHERE status IN ('RUNNING', 'CLAIMED') AND last_heartbeat_at < :cutoff """) with engine_core.connect() as conn: result = conn.execute(stmt, {"cutoff": cutoff}) conn.commit() reaped_count = result.rowcount except Exception as e: print(f"[Swarm Recovery] Stale job reap notice: {e}") return reaped_count