119 lines
3.9 KiB
Python
119 lines
3.9 KiB
Python
"""
|
|
Unit Tests: Durable Job Queue, State Transitions & Zombie Recovery
|
|
"""
|
|
import sys
|
|
sys.path.insert(0, '/mnt/fam/SERVER/IfixKartEcommerce/Backend')
|
|
sys.path.insert(0, '/mnt/fam/SERVER/IfixKartEcommerce')
|
|
from dotenv import load_dotenv
|
|
load_dotenv('/mnt/fam/SERVER/IfixKartEcommerce/Backend/.env')
|
|
|
|
import unittest
|
|
import asyncio
|
|
import ulid
|
|
from datetime import datetime, timedelta
|
|
from app.core.database.db_session import SessionLocal
|
|
from swarm.models.simulation_job import SimulationJob
|
|
from swarm.queue import (
|
|
execute_simulation_job,
|
|
request_job_cancellation,
|
|
is_cancellation_requested,
|
|
update_job_heartbeat,
|
|
reap_stale_jobs
|
|
)
|
|
|
|
|
|
class TestQueueAndRecovery(unittest.TestCase):
|
|
def test_job_execution_lifecycle(self):
|
|
async def _run():
|
|
session = SessionLocal()
|
|
job_id = str(ulid.ULID())
|
|
job = SimulationJob(
|
|
id=job_id,
|
|
store_id=1,
|
|
user_id=1,
|
|
scenario_type="pricing_elasticity",
|
|
status="QUEUED",
|
|
input_params={"discount_pct": 10.0, "current_price": 3000.0, "cost_price": 1800.0}
|
|
)
|
|
session.add(job)
|
|
session.commit()
|
|
session.close()
|
|
|
|
# Run job
|
|
await execute_simulation_job(job_id)
|
|
|
|
# Inspect final state
|
|
session = SessionLocal()
|
|
completed_job = session.query(SimulationJob).filter(SimulationJob.id == job_id).first()
|
|
self.assertIsNotNone(completed_job)
|
|
self.assertEqual(completed_job.status, "COMPLETED")
|
|
self.assertIsNotNone(completed_job.report_markdown)
|
|
self.assertIn("EPISTEMIC NOTICE", completed_job.report_markdown)
|
|
self.assertIsNotNone(completed_job.summary_metrics)
|
|
self.assertIsNotNone(completed_job.completed_at)
|
|
session.delete(completed_job)
|
|
session.commit()
|
|
session.close()
|
|
|
|
asyncio.run(_run())
|
|
|
|
def test_job_cancellation(self):
|
|
session = SessionLocal()
|
|
job_id = str(ulid.ULID())
|
|
job = SimulationJob(
|
|
id=job_id,
|
|
store_id=1,
|
|
user_id=1,
|
|
scenario_type="workshop_surge",
|
|
status="QUEUED",
|
|
input_params={"surge_multiplier": 1.5}
|
|
)
|
|
session.add(job)
|
|
session.commit()
|
|
session.close()
|
|
|
|
# Request cancellation
|
|
cancelled = request_job_cancellation(job_id)
|
|
self.assertTrue(cancelled)
|
|
self.assertTrue(is_cancellation_requested(job_id))
|
|
|
|
session = SessionLocal()
|
|
cancelled_job = session.query(SimulationJob).filter(SimulationJob.id == job_id).first()
|
|
self.assertEqual(cancelled_job.status, "CANCELLED")
|
|
session.delete(cancelled_job)
|
|
session.commit()
|
|
session.close()
|
|
|
|
def test_zombie_reaper_recovery(self):
|
|
session = SessionLocal()
|
|
job_id = str(ulid.ULID())
|
|
old_time = datetime.utcnow() - timedelta(minutes=10)
|
|
job = SimulationJob(
|
|
id=job_id,
|
|
store_id=1,
|
|
user_id=1,
|
|
scenario_type="whatsapp_preflight",
|
|
status="RUNNING",
|
|
worker_id="crashed-worker-old",
|
|
last_heartbeat_at=old_time,
|
|
input_params={"frequency": 3}
|
|
)
|
|
session.add(job)
|
|
session.commit()
|
|
session.close()
|
|
|
|
# Run reaper
|
|
reaped_count = reap_stale_jobs()
|
|
self.assertGreaterEqual(reaped_count, 1)
|
|
|
|
session = SessionLocal()
|
|
recovered_job = session.query(SimulationJob).filter(SimulationJob.id == job_id).first()
|
|
self.assertEqual(recovered_job.status, "FAILED_INTERRUPTED")
|
|
self.assertIn("interrupted by server crash", recovered_job.error_message)
|
|
session.delete(recovered_job)
|
|
session.commit()
|
|
session.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|