ifixkart-backend/swarm/tests/test_queue_and_recovery.py

119 lines
3.9 KiB
Python

"""
Unit Tests: Durable Job Queue, State Transitions & Zombie Recovery
"""
import sys
sys.path.insert(0, '/mnt/fam/SERVER/IfixKartEcommerce/Backend')
sys.path.insert(0, '/mnt/fam/SERVER/IfixKartEcommerce')
from dotenv import load_dotenv
load_dotenv('/mnt/fam/SERVER/IfixKartEcommerce/Backend/.env')
import unittest
import asyncio
import ulid
from datetime import datetime, timedelta
from app.core.database.db_session import SessionLocal
from swarm.models.simulation_job import SimulationJob
from swarm.queue import (
execute_simulation_job,
request_job_cancellation,
is_cancellation_requested,
update_job_heartbeat,
reap_stale_jobs
)
class TestQueueAndRecovery(unittest.TestCase):
def test_job_execution_lifecycle(self):
async def _run():
session = SessionLocal()
job_id = str(ulid.ULID())
job = SimulationJob(
id=job_id,
store_id=1,
user_id=1,
scenario_type="pricing_elasticity",
status="QUEUED",
input_params={"discount_pct": 10.0, "current_price": 3000.0, "cost_price": 1800.0}
)
session.add(job)
session.commit()
session.close()
# Run job
await execute_simulation_job(job_id)
# Inspect final state
session = SessionLocal()
completed_job = session.query(SimulationJob).filter(SimulationJob.id == job_id).first()
self.assertIsNotNone(completed_job)
self.assertEqual(completed_job.status, "COMPLETED")
self.assertIsNotNone(completed_job.report_markdown)
self.assertIn("EPISTEMIC NOTICE", completed_job.report_markdown)
self.assertIsNotNone(completed_job.summary_metrics)
self.assertIsNotNone(completed_job.completed_at)
session.delete(completed_job)
session.commit()
session.close()
asyncio.run(_run())
def test_job_cancellation(self):
session = SessionLocal()
job_id = str(ulid.ULID())
job = SimulationJob(
id=job_id,
store_id=1,
user_id=1,
scenario_type="workshop_surge",
status="QUEUED",
input_params={"surge_multiplier": 1.5}
)
session.add(job)
session.commit()
session.close()
# Request cancellation
cancelled = request_job_cancellation(job_id)
self.assertTrue(cancelled)
self.assertTrue(is_cancellation_requested(job_id))
session = SessionLocal()
cancelled_job = session.query(SimulationJob).filter(SimulationJob.id == job_id).first()
self.assertEqual(cancelled_job.status, "CANCELLED")
session.delete(cancelled_job)
session.commit()
session.close()
def test_zombie_reaper_recovery(self):
session = SessionLocal()
job_id = str(ulid.ULID())
old_time = datetime.utcnow() - timedelta(minutes=10)
job = SimulationJob(
id=job_id,
store_id=1,
user_id=1,
scenario_type="whatsapp_preflight",
status="RUNNING",
worker_id="crashed-worker-old",
last_heartbeat_at=old_time,
input_params={"frequency": 3}
)
session.add(job)
session.commit()
session.close()
# Run reaper
reaped_count = reap_stale_jobs()
self.assertGreaterEqual(reaped_count, 1)
session = SessionLocal()
recovered_job = session.query(SimulationJob).filter(SimulationJob.id == job_id).first()
self.assertEqual(recovered_job.status, "FAILED_INTERRUPTED")
self.assertIn("interrupted by server crash", recovered_job.error_message)
session.delete(recovered_job)
session.commit()
session.close()
if __name__ == "__main__":
unittest.main()