ifixkart-backend/swarm/queue/recovery.py

36 lines
1.4 KiB
Python

"""
@recovery Recovery (swarm/queue/recovery.py)
@purpose Heartbeat monitor & startup zombie job reaper for crash recovery.
"""
from datetime import datetime, timedelta
from sqlalchemy import text
from app.core.database.db_session import engine_core
from swarm.config import swarm_settings
def reap_stale_jobs() -> int:
"""
Identifies jobs left in 'RUNNING' or 'CLAIMED' state from an earlier server crash
(where last_heartbeat_at is older than ZOMBIE_REAP_MINUTES).
Transitions them safely and atomically to 'FAILED_INTERRUPTED' directly on engine_core.
Returns count of reaped jobs.
"""
cutoff = datetime.utcnow() - timedelta(minutes=swarm_settings.ZOMBIE_REAP_MINUTES)
reaped_count = 0
try:
stmt = text("""
UPDATE simulation_jobs
SET status = 'FAILED_INTERRUPTED',
error_message = 'Job interrupted by server crash or worker restart. No heartbeat detected within recovery window.',
completed_at = UTC_TIMESTAMP()
WHERE status IN ('RUNNING', 'CLAIMED')
AND last_heartbeat_at < :cutoff
""")
with engine_core.connect() as conn:
result = conn.execute(stmt, {"cutoff": cutoff})
conn.commit()
reaped_count = result.rowcount
except Exception as e:
print(f"[Swarm Recovery] Stale job reap notice: {e}")
return reaped_count