import os import io import csv import uuid import shutil import zipfile import threading import datetime from typing import List, Dict, Any, Optional from fastapi import APIRouter, UploadFile, File, Form, Depends, HTTPException, status from pydantic import BaseModel from fastapi.responses import Response, StreamingResponse from sqlalchemy import or_, text from sqlalchemy.orm import Session from app.core.database.db_session import get_db, SessionLocal import openpyxl from openpyxl.styles import Font, PatternFill, Alignment from openpyxl.utils import get_column_letter from app.models.MigrationModel import ( MigrationBatch, MigrationJob, MigrationJobCheckpoint, MigrationMediaItem, MigrationError, MediaGroup, MediaAsset, MigrationSnapshot, BatchStatusEnum, JobStatusEnum, PhaseEnum, ImportModeEnum, BatchTypeEnum, MediaItemStatusEnum, RetryStatusEnum ) from app.models.ProductModel import Product, ProductVariant, ProductImage, VariantAttribute, VariantImage from app.models.BrandModel import Brand from app.models.CategoryModel import Category from app.models.DeviceCatalogModel import DeviceSeries, DeviceModel from app.services.migration_engine.file_parsers import DataFileParser from app.services.migration_engine.column_mapper import ColumnMapper from app.services.migration_engine.storage_manager import StorageManager from app.services.migration_engine.migration_worker import MigrationWorker from app.core.permissions.RoleChecker import RoleChecker router = APIRouter( prefix="/api/v1/migration", tags=["Data Migration Engine"], dependencies=[Depends(RoleChecker(["Super Admin", "Admin"]))] ) from pathlib import Path curr = Path(__file__).resolve() backend_root = None for p in curr.parents: if (p / "app").is_dir(): backend_root = p break if not backend_root: backend_root = curr.parents[4] UPLOAD_DIR = str(backend_root / "uploads" / "migrations") os.makedirs(UPLOAD_DIR, exist_ok=True) # ----------------------------------------------------------------------------- # Background Worker Daemon Loop - Controlled centrally via main.py ENABLE_MIGRATION_WORKER # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- # Endpoints # ----------------------------------------------------------------------------- @router.get("/template/excel") def download_excel_template(db: Session = Depends(get_db)): """ Generates dynamic multi-sheet Excel spreadsheet (.xlsx) with styled headers and sample demo data. """ wb = openpyxl.Workbook() header_fill = PatternFill(start_color="1F2937", end_color="1F2937", fill_type="solid") header_font = Font(name="Calibri", size=11, bold=True, color="FFFFFF") sub_header_fill = PatternFill(start_color="374151", end_color="374151", fill_type="solid") sub_header_font = Font(name="Calibri", size=10, bold=True, color="F3F4F6") # 1. Products_and_Variants ws1 = wb.active ws1.title = "Products_and_Variants" ws1.views.sheetView[0].showGridLines = True headers_products = [ "sku", "parent_name", "name", "brand", "parent_category", "category", "is_parent_feature", "device_type", "device_series", "device_model", "price", "cost_price", "stock", "color", "storage", "ram", "material", "warranty_type", "warranty_summary", "parent_media_key", "media_key", "barcode", "is_active", "description", "seo_title", "seo_description", "meta_keywords", "show_specifications", "is_service_booking", "service_category", "store_locations" ] ws1.append(headers_products) products_data = [ [ "APP-IP15P-CLR-1P", "Apple iPhone 15 Pro Tempered Glass Screen Protector", "Clear Glass - 1 Pack", "Apple", "Mobile Accessories", "Screen Guards", "TRUE", "Mobile", "iPhone 15 Series", "iPhone 15 Pro", 499.00, 180.00, 150, "Clear", "N/A", "N/A", "9H Tempered Glass", "Brand Warranty", "6 Months Brand Replacement Warranty", "apple_iphone_15_pro_screenguard", "apple_iphone_15_pro_screenguard_clear_1p", "8901234567890", "TRUE", '

Features

', '

Buy Apple iPhone 15 Pro Screen Protector | iFixKart

', '

Premium 9H tempered glass screen protector for iPhone 15 Pro.

', '

screen guard, tempered glass, mobile accessories

', "TRUE", "FALSE", "", "CHENNAI:100;TEEST:50" ], [ "SAM-S24U-ARM-BLK", "Samsung Galaxy S24 Ultra Heavy Duty Armor Case", "Matte Black Shield", "Samsung", "Mobile Accessories", "Back Covers", "TRUE", "Mobile", "Galaxy S Series", "Galaxy S24 Ultra", 999.00, 380.00, 75, "Matte Black", "N/A", "N/A", "TPU + Polycarbonate", "Brand Warranty", "12 Months Brand Replacement Warranty", "samsung_s24_ultra_case", "samsung_s24_ultra_case_black", "8901234567892", "TRUE", '

Armor Protection

', 'Samsung Galaxy S24 Ultra Heavy Duty Armor Case | iFixKart', '

Shop shockproof armor cover for Samsung Galaxy S24 Ultra.

', 'samsung case, armor cover, mobile accessories', "TRUE", "FALSE", "", "CHENNAI:50;TEEST:25" ], [ "APP-IP15P-DISP-OEM", "Display with Touch Screen for Apple iPhone 15 Pro", "Display Combo - Black Titanium", "Apple", "Mobile Spare Parts", "Displays & Touch Screens", "TRUE", "Mobile", "iPhone 15 Series", "iPhone 15 Pro", 12500.00, 8200.00, 35, "Black Titanium", "N/A", "N/A", "OLED Display Combo", "Brand Warranty", "6 Months Screen Replacement Warranty", "apple_iphone_15_pro_display", "apple_iphone_15_pro_display_black", "8901234567899", "TRUE", '

OEM Grade Super Retina OLED Screen replacement assembly for iPhone 15 Pro.

', 'Apple iPhone 15 Pro OLED Display Screen Replacement | iFixKart', 'Order premium OLED Display screen replacement combo for Apple iPhone 15 Pro with door-step service.', 'iphone 15 pro screen, display replacement, oled combo', "TRUE", "TRUE", "Screen Replacement", "CHENNAI:25:12500;TEEST:10:12500" ] ] for row in products_data: ws1.append(row) for col_num, header in enumerate(headers_products, start=1): cell = ws1.cell(row=1, column=col_num) cell.fill = header_fill cell.font = header_font cell.alignment = Alignment(horizontal="center", vertical="center") output = io.BytesIO() wb.save(output) output.seek(0) return Response( content=output.getvalue(), media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", headers={"Content-Disposition": "attachment; filename=iFixKart_Bulk_Import_Demo_Template.xlsx"} ) @router.get("/template/zip") def download_zip_template(): """ Generates sample Media ZIP archive template. """ output = io.BytesIO() sample_image_bytes = ( b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01' b'\x08\x02\x00\x00\x00\x90wS\xde\x00\x00\x00\x0cIDATx\x9cc\xf8\xcf\xc0' b'\x00\x00\x03\x01\x01\x00\x18\xdd\x8d\xb0\x00\x00\x00\x00IEND\xaeB`\x82' ) with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as zf: zf.writestr("README.txt", "iFixKart Media ZIP Template") zf.writestr("apple_iphone_15_pro_screenguard/1_front.png", sample_image_bytes) zf.writestr("samsung_s24_ultra_case/1_main.png", sample_image_bytes) output.seek(0) return Response( content=output.getvalue(), media_type="application/zip", headers={"Content-Disposition": "attachment; filename=iFixKart_Media_ZIP_Demo_Template.zip"} ) from app.services.migration_engine.validation_engine import ValidationEngine @router.post("/preview") @router.get("/preview") def preview_migration_data(job_id: Optional[str] = None, db: Session = Depends(get_db)): """ Dynamic preview endpoint returning mapped file columns and sample rows. """ job = None if job_id: job = db.query(MigrationJob).filter(MigrationJob.id == job_id).first() if not job: job = db.query(MigrationJob).order_by(MigrationJob.started_at.desc(), MigrationJob.id.desc()).first() if not job: return {"status": "success", "valid": True, "preview_items": []} job_dir = StorageManager.get_job_dir(job.id) dataset_dir = os.path.join(job_dir, "dataset") if not os.path.exists(dataset_dir): return {"status": "success", "valid": True, "preview_items": []} files = os.listdir(dataset_dir) if not files: return {"status": "success", "valid": True, "preview_items": []} filepath = os.path.join(dataset_dir, files[0]) fmt = os.path.splitext(filepath)[1].lower().replace(".", "").upper() headers = DataFileParser.get_headers(filepath, fmt) column_maps = ColumnMapper.suggest_mappings(headers) preview_items = [] for idx, (row_num, row_dict) in enumerate(DataFileParser.stream_rows(filepath, fmt)): if idx >= 10: break mapped = ColumnMapper.apply_mapping(row_dict, column_maps) preview_items.append(mapped) return { "status": "success", "job_id": job.id, "valid": True, "headers": headers, "column_maps": column_maps, "preview_items": preview_items } @router.get("/media-groups/preview") @router.post("/media-groups/preview") def get_media_groups_preview(db: Session = Depends(get_db)): """ Returns indexed MediaGroup records with their assets for the Media Library tab. """ groups = db.query(MediaGroup).order_by(MediaGroup.created_at.desc()).all() res = [] for g in groups: assets = [] for a in g.media_assets: assets.append({ "id": a.id, "original_filename": a.original_filename, "cdn_url": a.cdn_url, "thumbnail_url": a.thumbnail_url, "file_size_bytes": a.file_size_bytes, "mime_type": a.mime_type }) res.append({ "id": g.id, "media_key": g.media_key, "brand_name": g.brand_name, "model_name": g.model_name, "variant_tag": g.variant_tag, "assets_count": len(assets), "media_assets": assets }) return { "status": "success", "total_count": len(res), "media_groups": res } @router.post("/dry-run") @router.get("/dry-run") @router.post("/dry_run") @router.get("/dry_run") def dry_run_migration_data(job_id: Optional[str] = None, db: Session = Depends(get_db)): """ Dynamic dry-run endpoint executing real ValidationEngine checks against uploaded file rows. """ job = None if job_id: job = db.query(MigrationJob).filter(MigrationJob.id == job_id).first() if not job: job = db.query(MigrationJob).order_by(MigrationJob.started_at.desc(), MigrationJob.id.desc()).first() if not job: return { "status": "success", "message": "No active migration job found to validate.", "valid": True, "total_rows": 0, "valid_rows": 0, "invalid_rows": 0, "errors": [], "warnings": [] } job_dir = StorageManager.get_job_dir(job.id) dataset_dir = os.path.join(job_dir, "dataset") if not os.path.exists(dataset_dir) or not os.listdir(dataset_dir): return { "status": "success", "message": f"Job {job.id} initialized. Dataset file pending.", "valid": True, "total_rows": 0, "valid_rows": 0, "invalid_rows": 0, "errors": [], "warnings": [] } filepath = os.path.join(dataset_dir, os.listdir(dataset_dir)[0]) fmt = os.path.splitext(filepath)[1].lower().replace(".", "").upper() headers = DataFileParser.get_headers(filepath, fmt) column_maps = ColumnMapper.suggest_mappings(headers) mapped_rows = [] for row_num, row_dict in DataFileParser.stream_rows(filepath, fmt): mapped = ColumnMapper.apply_mapping(row_dict, column_maps) mapped_rows.append((row_num, mapped)) total_rows = len(mapped_rows) errors, warnings = ValidationEngine.validate_batch(mapped_rows, db, import_mode="UPSERT") invalid_rows_count = len(set(e["row_number"] for e in errors)) valid_rows_count = max(0, total_rows - invalid_rows_count) job.total_records = total_rows db.commit() return { "status": "success", "job_id": job.id, "message": f"Dynamic Dry-Run Validation complete for {total_rows} rows.", "valid": invalid_rows_count == 0, "total_rows": total_rows, "valid_rows": valid_rows_count, "invalid_rows": invalid_rows_count, "errors": errors[:100], "warnings": warnings[:100], "headers": headers, "column_maps": column_maps } @router.post("/execute") @router.get("/execute") def execute_migration_batch(job_id: Optional[str] = None, db: Session = Depends(get_db)): """ Dynamic execution endpoint enqueuing job for persistent MigrationWorker processing. """ job = None if job_id: job = db.query(MigrationJob).filter(MigrationJob.id == job_id).first() if not job: job = db.query(MigrationJob).order_by(MigrationJob.started_at.desc(), MigrationJob.id.desc()).first() if not job: raise HTTPException(status_code=404, detail="No migration job found to execute.") # Only mark as QUEUED if the job is not already RUNNING or COMPLETED if job.status not in (JobStatusEnum.RUNNING, JobStatusEnum.COMPLETED): job.status = JobStatusEnum.QUEUED if job.current_phase == PhaseEnum.UPLOAD: job.current_phase = PhaseEnum.VALIDATE db.commit() return { "status": "success", "job_id": job.id, "batch_id": job.batch_id, "job_status": job.status, "message": f"Migration Job {job.id} enqueued. Background worker thread will execute pipeline phases." } @router.post("/upload") async def upload_migration_file( file: Optional[UploadFile] = File(None), media_file: Optional[UploadFile] = File(None), batch_type: str = Form("PRODUCTS"), import_mode: str = Form("UPSERT"), media_structure: str = Form("AUTO"), current_user: Any = Depends(RoleChecker(["Super Admin", "Admin"])), db: Session = Depends(get_db) ): """ 64KB Streamed Upload Endpoint. Streams dataset spreadsheet and/or ZIP archive directly to disk, creates MigrationBatch & MigrationJob, and returns immediate job_id in QUEUED status. Guarded with role authentication and directory traversal protection. """ import re if not file and not media_file: raise HTTPException(status_code=400, detail="Please upload a dataset file (.xlsx/.csv) or media ZIP archive (.zip).") batch_id = str(uuid.uuid4()) job_id = str(uuid.uuid4()) job_dir = os.path.abspath(StorageManager.get_job_dir(job_id)) # Save job configuration including media_structure (AUTO vs FLAT) StorageManager.save_job_config(job_id, {"media_structure": media_structure, "import_mode": import_mode}) batch = MigrationBatch( id=batch_id, batch_type=BatchTypeEnum(batch_type) if batch_type in BatchTypeEnum.__members__ else BatchTypeEnum.PRODUCTS, user_id=getattr(current_user, "user_id", "admin"), import_mode=ImportModeEnum(import_mode) if import_mode in ImportModeEnum.__members__ else ImportModeEnum.UPSERT, status=BatchStatusEnum.PENDING ) db.add(batch) file_name = "" file_format = "CSV" if file: raw_name = os.path.basename(file.filename or "dataset.csv") clean_name = re.sub(r"[^a-zA-Z0-9._\-]", "_", raw_name) ext = os.path.splitext(clean_name)[1].lower().replace(".", "") if ext not in ("csv", "xlsx", "xls"): raise HTTPException(status_code=400, detail="Unsupported dataset file format. Use .xlsx or .csv") dataset_dir = os.path.abspath(os.path.join(job_dir, "dataset")) dataset_dest = os.path.abspath(os.path.join(dataset_dir, clean_name)) if not dataset_dest.startswith(dataset_dir + os.sep) and dataset_dest != dataset_dir: raise HTTPException(status_code=400, detail="Path traversal attempt in dataset filename") file_name = clean_name file_format = ext.upper() await StorageManager.save_upload_stream_async(file, dataset_dest) if media_file: raw_archive = os.path.basename(media_file.filename or "media.zip") clean_archive = re.sub(r"[^a-zA-Z0-9._\-]", "_", raw_archive) if not clean_archive.lower().endswith(".zip"): raise HTTPException(status_code=400, detail="Media archive must be a .zip file") archive_dir = os.path.abspath(os.path.join(job_dir, "archives")) archive_dest = os.path.abspath(os.path.join(archive_dir, clean_archive)) if not archive_dest.startswith(archive_dir + os.sep) and archive_dest != archive_dir: raise HTTPException(status_code=400, detail="Path traversal attempt in archive filename") await StorageManager.save_upload_stream_async(media_file, archive_dest) if not file_name: file_name = clean_archive file_format = "ZIP" job = MigrationJob( id=job_id, batch_id=batch_id, job_type=batch_type, is_dry_run=False, file_name=file_name, file_format=file_format, status=JobStatusEnum.QUEUED, current_phase=PhaseEnum.UPLOAD ) db.add(job) db.commit() return { "status": "success", "job_id": job.id, "batch_id": batch.id, "job_status": job.status, "current_phase": job.current_phase, "message": f"File uploaded safely to disk. Migration Job {job.id} queued for background processing." } @router.get("/batches") @router.get("/jobs") @router.get("/history") def list_migration_jobs(db: Session = Depends(get_db)): """ List all historical migration jobs for the admin dashboard. """ jobs = db.query(MigrationJob).order_by(MigrationJob.started_at.desc(), MigrationJob.id.desc()).all() res = [] for j in jobs: dur_secs = 0 if j.started_at: end_t = j.completed_at or j.finished_at or datetime.datetime.utcnow() dur_secs = max(0, int((end_t - j.started_at).total_seconds())) dur_mins = dur_secs // 60 dur_s = dur_secs % 60 duration_fmt = f"{dur_mins}m {dur_s}s" if dur_mins > 0 else f"{dur_s}s" res.append({ "id": j.id, "job_id": j.id, "batch_id": j.batch_id, "file_name": j.file_name, "file_format": j.file_format, "status": j.status, "current_phase": j.current_phase, "current_batch": j.current_batch, "total_batches": j.total_batches, "processed_records": j.processed_records, "total_records": j.total_records, "successful_records": j.successful_records, "failed_records": j.failed_records, "worker_id": j.worker_id, "heartbeat_at": j.heartbeat_at.isoformat() if j.heartbeat_at else None, "started_at": j.started_at.strftime("%Y-%m-%d %H:%M:%S") if j.started_at else "Pending", "completed_at": j.completed_at.strftime("%Y-%m-%d %H:%M:%S") if j.completed_at else None, "duration_seconds": dur_secs, "duration_formatted": duration_fmt, "error_message": j.error_message }) return {"status": "success", "batches": res, "jobs": res} class BulkDeleteMediaRequest(BaseModel): ids: Optional[list[str]] = None group_ids: Optional[list[str]] = None def target_ids(self) -> list[str]: combined = [] if self.ids: combined.extend(self.ids) if self.group_ids: combined.extend(self.group_ids) return list(dict.fromkeys(str(x).strip() for x in combined if x and str(x).strip())) @router.delete("/media-groups/{group_id}") def delete_single_media_group(group_id: str, db: Session = Depends(get_db)): """ Deletes a MediaGroup along with its MediaAssets, linked variant/product images, and disk files. Accepts MediaGroup id, media_key, or staging MigrationMediaItem id. """ # 1. Search for MediaGroup by id or media_key mg = db.query(MediaGroup).filter(or_(MediaGroup.id == group_id, MediaGroup.media_key == group_id)).first() # 2. If not found, check if group_id is a MigrationMediaItem id if not mg: item = db.query(MigrationMediaItem).filter(MigrationMediaItem.id == group_id).first() if item and item.media_key: mg = db.query(MediaGroup).filter(MediaGroup.media_key == item.media_key).first() try: db.delete(item) except Exception: pass if mg: media_key = mg.media_key # Delete physical files and collect URLs to unlink asset_urls = [] for asset in (mg.media_assets or []): if asset.storage_path and os.path.exists(asset.storage_path): try: os.remove(asset.storage_path) except Exception: pass if asset.cdn_url: asset_urls.append(asset.cdn_url) # Unlink from VariantImage and ProductImage if asset_urls: db.query(VariantImage).filter(VariantImage.image_url.in_(asset_urls)).delete(synchronize_session=False) db.query(ProductImage).filter(ProductImage.image_url.in_(asset_urls)).delete(synchronize_session=False) # Delete any staging media items matching this key if media_key: db.query(MigrationMediaItem).filter( or_(MigrationMediaItem.media_key == media_key, MigrationMediaItem.id == group_id) ).delete(synchronize_session=False) # Delete media group (cascades to MediaAsset) db.delete(mg) db.commit() return {"status": "success", "message": f"Media item {media_key} deleted successfully."} # Fallback: check if staging items exist deleted_items = db.query(MigrationMediaItem).filter( or_(MigrationMediaItem.id == group_id, MigrationMediaItem.media_key == group_id) ).delete(synchronize_session=False) db.commit() if deleted_items > 0: return {"status": "success", "message": f"Media item {group_id} deleted."} return {"status": "success", "message": f"Media item {group_id} removed."} @router.post("/media-groups/bulk-delete") def bulk_delete_media_groups(payload: BulkDeleteMediaRequest, db: Session = Depends(get_db)): """ Bulk deletes multiple MediaGroup records, their MediaAssets, linked variant/product images, and disk files. Supports both payload.ids and payload.group_ids. """ target_ids = payload.target_ids() if not target_ids: return {"status": "success", "message": "No media keys specified for deletion.", "deleted_groups_count": 0} # Find all MediaGroups matching IDs or media_keys groups = db.query(MediaGroup).filter( or_(MediaGroup.id.in_(target_ids), MediaGroup.media_key.in_(target_ids)) ).all() media_keys = [g.media_key for g in groups if g.media_key] # Also check if any target_ids are MigrationMediaItem IDs staging_items = db.query(MigrationMediaItem).filter(MigrationMediaItem.id.in_(target_ids)).all() staging_keys = [i.media_key for i in staging_items if i.media_key] if staging_keys: more_groups = db.query(MediaGroup).filter(MediaGroup.media_key.in_(staging_keys)).all() for mg in more_groups: if mg not in groups: groups.append(mg) if mg.media_key: media_keys.append(mg.media_key) # Clean up physical files and collect asset URLs all_cdn_urls = [] for g in groups: for asset in (g.media_assets or []): if asset.storage_path and os.path.exists(asset.storage_path): try: os.remove(asset.storage_path) except Exception: pass if asset.cdn_url: all_cdn_urls.append(asset.cdn_url) db.delete(g) # Unlink images from variants and products if all_cdn_urls: db.query(VariantImage).filter(VariantImage.image_url.in_(all_cdn_urls)).delete(synchronize_session=False) db.query(ProductImage).filter(ProductImage.image_url.in_(all_cdn_urls)).delete(synchronize_session=False) # Clean up staging items all_clean_keys = list(set(target_ids + media_keys + staging_keys)) db.query(MigrationMediaItem).filter( or_(MigrationMediaItem.id.in_(all_clean_keys), MigrationMediaItem.media_key.in_(all_clean_keys)) ).delete(synchronize_session=False) db.commit() count = len(groups) return {"status": "success", "message": f"Deleted {count} media key(s).", "deleted_groups_count": count} @router.post("/rollback/{batch_id}") def rollback_migration_batch(batch_id: str, db: Session = Depends(get_db)): """ Reverts inserted and updated records for a migration batch using migration_snapshots. """ batch = db.query(MigrationBatch).filter(MigrationBatch.id == batch_id).first() if not batch: raise HTTPException(status_code=404, detail="Migration batch not found") snapshots = db.query(MigrationSnapshot).filter( MigrationSnapshot.batch_id == batch_id ).order_by(MigrationSnapshot.created_at.desc()).all() reverted_inserts = 0 reverted_updates = 0 for snap in snapshots: if snap.entity_table == "product_variants": variant = db.query(ProductVariant).filter(ProductVariant.variant_id == snap.entity_id).first() if not variant: continue if snap.action_taken == "INSERTED": # Delete variant attributes & images db.query(VariantAttribute).filter(VariantAttribute.variant_id == variant.variant_id).delete(synchronize_session=False) db.query(VariantImage).filter(VariantImage.variant_id == variant.variant_id).delete(synchronize_session=False) prod_id = variant.product_id db.delete(variant) reverted_inserts += 1 # Clean up empty parent product if prod_id: remaining = db.query(ProductVariant).filter(ProductVariant.product_id == prod_id).count() if remaining <= 1: prod = db.query(Product).filter(Product.product_id == prod_id).first() if prod: db.query(ProductImage).filter(ProductImage.product_id == prod_id).delete(synchronize_session=False) db.delete(prod) elif snap.action_taken == "UPDATED" and snap.previous_state: prev = snap.previous_state if "price" in prev: variant.price = prev["price"] if "cost_price" in prev: variant.cost_price = prev["cost_price"] if "sku" in prev: variant.sku = prev["sku"] reverted_updates += 1 batch.status = BatchStatusEnum.ROLLED_BACK db.commit() return { "status": "success", "message": f"Rollback complete: {reverted_inserts} inserted variants removed, {reverted_updates} updated variants restored." } @router.post("/purge-all") @router.post("/purge_all") def purge_all_migration_data(db: Session = Depends(get_db)): try: db.execute(text("DELETE FROM migration_snapshots")) db.execute(text("DELETE FROM migration_errors")) db.execute(text("DELETE FROM migration_job_checkpoints")) db.execute(text("DELETE FROM migration_media_items")) db.execute(text("DELETE FROM media_library")) db.execute(text("DELETE FROM media_groups")) db.execute(text("DELETE FROM migration_jobs")) db.execute(text("DELETE FROM migration_batches")) db.commit() except Exception as e: db.rollback() print(f"Purge warning: {e}") return {"status": "success", "message": "All migration history, jobs, and media groups purged."} @router.get("/jobs/{job_id}/status") def get_job_telemetry(job_id: str, db: Session = Depends(get_db)): """ Fetches real-time telemetry metrics for a migration job. """ job = db.query(MigrationJob).filter(MigrationJob.id == job_id).first() if not job: raise HTTPException(status_code=404, detail="Migration job not found") checkpoints = db.query(MigrationJobCheckpoint).filter(MigrationJobCheckpoint.job_id == job_id).all() phase_checkpoints = {} for c in checkpoints: phase_checkpoints[c.phase] = { "last_successful_batch": c.last_successful_batch, "total_batches": c.total_batches, "processed_records": c.processed_records, "failed_records": c.failed_records } pct = 0.0 if job.total_records > 0: pct = round((job.processed_records / job.total_records) * 100.0, 2) elapsed_seconds = 0 if job.started_at: end_time = job.completed_at or job.finished_at or datetime.datetime.utcnow() elapsed_seconds = max(0, int((end_time - job.started_at).total_seconds())) processing_rate = 0.0 estimated_seconds_remaining = None if job.status == JobStatusEnum.QUEUED: eta_formatted = "Queued in Line" elif job.status == JobStatusEnum.RUNNING and job.processed_records == 0: eta_formatted = "Starting Batch Processing..." elif job.status == JobStatusEnum.COMPLETED: eta_formatted = "0s (Completed)" else: eta_formatted = "Calculating..." if elapsed_seconds > 0 and job.processed_records > 0: processing_rate = round(job.processed_records / elapsed_seconds, 2) remaining_records = max(0, job.total_records - job.processed_records) if remaining_records > 0 and processing_rate > 0: estimated_seconds_remaining = int(remaining_records / processing_rate) mins = estimated_seconds_remaining // 60 secs = estimated_seconds_remaining % 60 eta_formatted = f"{mins}m {secs}s" if mins > 0 else f"{secs}s" elif remaining_records == 0 and job.status == JobStatusEnum.COMPLETED: eta_formatted = "0s (Completed)" elapsed_mins = elapsed_seconds // 60 elapsed_secs = elapsed_seconds % 60 elapsed_formatted = f"{elapsed_mins}m {elapsed_secs}s" if elapsed_mins > 0 else f"{elapsed_secs}s" return { "status": "success", "job_id": job.id, "batch_id": job.batch_id, "job_status": job.status, "current_phase": job.current_phase, "worker_id": job.worker_id, "heartbeat_at": job.heartbeat_at.isoformat() if job.heartbeat_at else None, "lease_version": job.lease_version, "current_batch": job.current_batch, "total_batches": job.total_batches, "last_successful_batch": job.last_successful_batch, "processed_records": job.processed_records, "total_records": job.total_records, "successful_records": job.successful_records, "failed_records": job.failed_records, "expected_products": job.expected_products, "expected_variants": job.expected_variants, "expected_media_items": job.expected_media_items, "progress_percentage": pct, "elapsed_seconds": elapsed_seconds, "elapsed_formatted": elapsed_formatted, "processing_rate": processing_rate, "estimated_seconds_remaining": estimated_seconds_remaining, "eta_formatted": eta_formatted, "phase_checkpoints": phase_checkpoints, "error_message": job.error_message } @router.post("/sse-ticket") def create_sse_ticket( current_user: Any = Depends(RoleChecker(["Super Admin", "Admin"])), ): """ Issues a short-lived (30-second) single-use authorization ticket for EventSource SSE streaming, allowing browser EventSource to authenticate without exposing long-lived JWTs in URL query parameters. """ from app.core.database.cache_manager import cache ticket_id = f"sse_{uuid.uuid4().hex}" user_id = getattr(current_user, "user_id", "admin") cache.set(f"sse_ticket:{ticket_id}", user_id, ttl_seconds=30) return {"ticket": ticket_id, "expires_in": 30} @router.get("/jobs/{job_id}/stream") def stream_job_telemetry( job_id: str, ticket: Optional[str] = None, token: Optional[str] = None, ): """ Streams real-time Server-Sent Events (SSE) telemetry data for a migration job. Authenticates via single-use 30s SSE ticket (preferred) or token fallback. """ from app.core.database.cache_manager import cache from app.core.Token import verify_access_token is_authenticated = False # 1. Check single-use SSE ticket (burn immediately upon first validation) if ticket: cached_user = cache.get(f"sse_ticket:{ticket}") if cached_user: cache.delete(f"sse_ticket:{ticket}") is_authenticated = True # 2. Backward compatibility fallback for active sessions if not is_authenticated and token: try: payload = verify_access_token(token) if payload and payload.get("sub"): is_authenticated = True except Exception: pass if not is_authenticated: raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Unauthorized SSE connection: Valid ticket or token is required." ) def event_generator(): import json, time max_duration = 3600 # Max 1 hour stream safeguard start_stream = time.time() while time.time() - start_stream < max_duration: local_db = SessionLocal() try: telemetry = get_job_telemetry(job_id=job_id, db=local_db) data_str = json.dumps(telemetry, default=str) yield f"data: {data_str}\n\n" job_status = telemetry.get("job_status") if job_status in [JobStatusEnum.COMPLETED, JobStatusEnum.FAILED, JobStatusEnum.CANCELLED, "COMPLETED", "FAILED", "CANCELLED"]: break except Exception as e: err_payload = json.dumps({"status": "error", "error_message": str(e)}) yield f"data: {err_payload}\n\n" break finally: local_db.close() time.sleep(1.0) return StreamingResponse(event_generator(), media_type="text/event-stream") @router.post("/jobs/{job_id}/cancel") def cancel_job(job_id: str, db: Session = Depends(get_db)): """ Flags job status as CANCELLING. Worker finishes active batch transaction before halting cleanly. """ job = db.query(MigrationJob).filter(MigrationJob.id == job_id).first() if not job: raise HTTPException(status_code=404, detail="Migration job not found") if job.status in (JobStatusEnum.COMPLETED, JobStatusEnum.CANCELLED, JobStatusEnum.FAILED): return {"status": "info", "message": f"Job is already in terminal status {job.status}"} job.status = JobStatusEnum.CANCELLING job.cancel_requested_at = datetime.datetime.utcnow() db.commit() return {"status": "success", "message": f"Cancellation requested for job {job_id}. Worker will halt cleanly after current batch finishes."} @router.post("/jobs/{job_id}/resume") def resume_job(job_id: str, db: Session = Depends(get_db)): """ Resumes an interrupted, cancelled, or failed job from its last successful phase checkpoint. """ job = db.query(MigrationJob).filter(MigrationJob.id == job_id).first() if not job: raise HTTPException(status_code=404, detail="Migration job not found") if job.status == JobStatusEnum.RUNNING: return {"status": "info", "message": "Job is already running."} job.status = JobStatusEnum.QUEUED job.cancel_requested_at = None job.error_message = None db.commit() return {"status": "success", "message": f"Job {job_id} re-queued and will resume from phase {job.current_phase} batch {job.last_successful_batch}."} @router.get("/jobs/{job_id}/errors/export") def export_job_errors(job_id: str, db: Session = Depends(get_db)): """ Exports CSV failure report: Row Number, SKU, Product Name, File Name, Phase, Error Type, Error Message, Retry Status. """ job = db.query(MigrationJob).filter(MigrationJob.id == job_id).first() if not job: raise HTTPException(status_code=404, detail="Migration job not found") errors = db.query(MigrationError).filter(MigrationError.job_id == job_id).order_by(MigrationError.row_number.asc()).all() output = io.StringIO() writer = csv.writer(output) writer.writerow([ "Row Number", "SKU", "Product Name", "File Name", "Entity Type", "Phase", "Error Type", "Severity", "Error Message", "Suggested Fix", "Retry Status" ]) for e in errors: writer.writerow([ e.row_number, e.sku or "", e.product_name or "", e.file_name or "", e.entity_type or "", e.phase, e.error_type, e.severity, e.error_message, e.suggested_fix or "", e.retry_status ]) output.seek(0) return StreamingResponse( io.BytesIO(output.getvalue().encode("utf-8")), media_type="text/csv", headers={"Content-Disposition": f"attachment; filename=Migration_Errors_Job_{job_id[:8]}.csv"} ) @router.get("/jobs/{job_id}/media") def list_job_media( job_id: str, page: int = 1, limit: int = 50, search: Optional[str] = None, db: Session = Depends(get_db) ): """ Server-side paginated media gallery for admin console to view 30,000+ indexed media assets without DOM lag. """ query = db.query(MigrationMediaItem).filter(MigrationMediaItem.job_id == job_id) if search: query = query.filter( (MigrationMediaItem.file_name.ilike(f"%{search}%")) | (MigrationMediaItem.media_key.ilike(f"%{search}%")) | (MigrationMediaItem.sha256.ilike(f"%{search}%")) ) total_count = query.count() items = query.offset((page - 1) * limit).limit(limit).all() res = [] for item in items: res.append({ "id": item.id, "batch_number": item.batch_number, "file_name": item.file_name, "archive_name": item.archive_name, "zip_entry_path": item.zip_entry_path, "media_key": item.media_key, "sha256": item.sha256, "storage_path": item.storage_path, "cdn_url": f"/uploads/migrations/{job_id}/media/{item.sha256[:2]}/{item.sha256}.jpg" if item.sha256 else None, "status": item.status, "error": item.error }) return { "status": "success", "total_count": total_count, "page": page, "limit": limit, "total_pages": (total_count // limit) + (1 if total_count % limit > 0 else 0), "items": res }