36 lines
1.4 KiB
Python
36 lines
1.4 KiB
Python
"""
|
|
@recovery Recovery (swarm/queue/recovery.py)
|
|
@purpose Heartbeat monitor & startup zombie job reaper for crash recovery.
|
|
"""
|
|
from datetime import datetime, timedelta
|
|
from sqlalchemy import text
|
|
from app.core.database.db_session import engine_core
|
|
from swarm.config import swarm_settings
|
|
|
|
|
|
def reap_stale_jobs() -> int:
|
|
"""
|
|
Identifies jobs left in 'RUNNING' or 'CLAIMED' state from an earlier server crash
|
|
(where last_heartbeat_at is older than ZOMBIE_REAP_MINUTES).
|
|
Transitions them safely and atomically to 'FAILED_INTERRUPTED' directly on engine_core.
|
|
Returns count of reaped jobs.
|
|
"""
|
|
cutoff = datetime.utcnow() - timedelta(minutes=swarm_settings.ZOMBIE_REAP_MINUTES)
|
|
reaped_count = 0
|
|
try:
|
|
stmt = text("""
|
|
UPDATE simulation_jobs
|
|
SET status = 'FAILED_INTERRUPTED',
|
|
error_message = 'Job interrupted by server crash or worker restart. No heartbeat detected within recovery window.',
|
|
completed_at = UTC_TIMESTAMP()
|
|
WHERE status IN ('RUNNING', 'CLAIMED')
|
|
AND last_heartbeat_at < :cutoff
|
|
""")
|
|
with engine_core.connect() as conn:
|
|
result = conn.execute(stmt, {"cutoff": cutoff})
|
|
conn.commit()
|
|
reaped_count = result.rowcount
|
|
except Exception as e:
|
|
print(f"[Swarm Recovery] Stale job reap notice: {e}")
|
|
|
|
return reaped_count
|