723 lines
28 KiB
Python
723 lines
28 KiB
Python
import os
|
|
import io
|
|
import csv
|
|
import uuid
|
|
import shutil
|
|
import zipfile
|
|
import threading
|
|
import datetime
|
|
from typing import List, Dict, Any, Optional
|
|
from fastapi import APIRouter, UploadFile, File, Form, Depends, HTTPException, status
|
|
from pydantic import BaseModel
|
|
from fastapi.responses import Response, StreamingResponse
|
|
from sqlalchemy.orm import Session
|
|
from app.core.database.db_session import get_db, SessionLocal
|
|
import openpyxl
|
|
from openpyxl.styles import Font, PatternFill, Alignment
|
|
from openpyxl.utils import get_column_letter
|
|
|
|
from app.models.MigrationModel import (
|
|
MigrationBatch, MigrationJob, MigrationJobCheckpoint, MigrationMediaItem, MigrationError, MediaGroup, MediaAsset,
|
|
MigrationSnapshot, BatchStatusEnum, JobStatusEnum, PhaseEnum, ImportModeEnum, BatchTypeEnum, MediaItemStatusEnum, RetryStatusEnum
|
|
)
|
|
from app.models.ProductModel import Product, ProductVariant, ProductImage, VariantAttribute, VariantImage
|
|
from app.models.BrandModel import Brand
|
|
from app.models.CategoryModel import Category
|
|
from app.models.DeviceCatalogModel import DeviceSeries, DeviceModel
|
|
from app.services.migration_engine.file_parsers import DataFileParser
|
|
from app.services.migration_engine.column_mapper import ColumnMapper
|
|
from app.services.migration_engine.storage_manager import StorageManager
|
|
from app.services.migration_engine.migration_worker import MigrationWorker
|
|
|
|
router = APIRouter(prefix="/api/v1/migration", tags=["Data Migration Engine"])
|
|
|
|
from pathlib import Path
|
|
project_root = Path(__file__).resolve().parents[4]
|
|
UPLOAD_DIR = str(project_root / "uploads" / "migrations")
|
|
os.makedirs(UPLOAD_DIR, exist_ok=True)
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Background Worker Daemon Loop
|
|
# -----------------------------------------------------------------------------
|
|
def worker_daemon_loop():
|
|
worker = MigrationWorker()
|
|
while True:
|
|
try:
|
|
db = SessionLocal()
|
|
try:
|
|
claimed = worker.claim_next_job(db)
|
|
if claimed:
|
|
job_id, lease_ver = claimed
|
|
worker.process_job(job_id, lease_ver)
|
|
finally:
|
|
db.close()
|
|
except Exception as e:
|
|
print(f"[WorkerDaemon] Error in worker loop: {e}")
|
|
time_to_sleep = 2.0
|
|
import time
|
|
time.sleep(time_to_sleep)
|
|
|
|
_worker_thread = threading.Thread(target=worker_daemon_loop, daemon=True)
|
|
_worker_thread.start()
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Endpoints
|
|
# -----------------------------------------------------------------------------
|
|
|
|
@router.get("/template/excel")
|
|
def download_excel_template(db: Session = Depends(get_db)):
|
|
"""
|
|
Generates dynamic multi-sheet Excel spreadsheet (.xlsx) with styled headers and sample demo data.
|
|
"""
|
|
wb = openpyxl.Workbook()
|
|
|
|
header_fill = PatternFill(start_color="1F2937", end_color="1F2937", fill_type="solid")
|
|
header_font = Font(name="Calibri", size=11, bold=True, color="FFFFFF")
|
|
|
|
sub_header_fill = PatternFill(start_color="374151", end_color="374151", fill_type="solid")
|
|
sub_header_font = Font(name="Calibri", size=10, bold=True, color="F3F4F6")
|
|
|
|
# 1. Products_and_Variants
|
|
ws1 = wb.active
|
|
ws1.title = "Products_and_Variants"
|
|
ws1.views.sheetView[0].showGridLines = True
|
|
|
|
headers_products = [
|
|
"sku", "parent_name", "name", "brand", "parent_category", "category",
|
|
"is_parent_feature", "device_type", "device_series", "device_model",
|
|
"price", "cost_price", "stock", "color", "storage", "ram", "material",
|
|
"warranty_type", "warranty_summary", "parent_media_key", "media_key", "barcode", "is_active",
|
|
"description", "seo_title", "seo_description", "meta_keywords"
|
|
]
|
|
ws1.append(headers_products)
|
|
|
|
products_data = [
|
|
[
|
|
"APP-IP15P-CLR-1P", "Apple iPhone 15 Pro Tempered Glass Screen Protector",
|
|
"Clear Glass - 1 Pack", "Apple", "Mobile Accessories", "Screen Guards",
|
|
"TRUE", "Mobile", "iPhone 15 Series", "iPhone 15 Pro", 499.00, 180.00,
|
|
150, "Clear", "N/A", "N/A", "9H Tempered Glass", "Brand Warranty", "6 Months Brand Replacement Warranty",
|
|
"apple_iphone_15_pro_screenguard", "apple_iphone_15_pro_screenguard_clear_1p", "8901234567890", "TRUE",
|
|
'<p style="text-align: center;"><strong>Features</strong></p><ul><li>100% brand new combo</li><li>9H tempered glass</li></ul>',
|
|
'<p><strong>Buy Apple iPhone 15 Pro Screen Protector</strong> | iFixKart</p>',
|
|
'<p>Premium <strong>9H tempered glass</strong> screen protector for iPhone 15 Pro.</p>',
|
|
'<p>screen guard, tempered glass, mobile accessories</p>'
|
|
],
|
|
[
|
|
"SAM-S24U-ARM-BLK", "Samsung Galaxy S24 Ultra Heavy Duty Armor Case",
|
|
"Matte Black Shield", "Samsung", "Mobile Accessories", "Back Covers",
|
|
"TRUE", "Mobile", "Galaxy S Series", "Galaxy S24 Ultra", 999.00, 380.00,
|
|
75, "Matte Black", "N/A", "N/A", "TPU + Polycarbonate", "Brand Warranty", "12 Months Brand Replacement Warranty",
|
|
"samsung_s24_ultra_case", "samsung_s24_ultra_case_black", "8901234567892", "TRUE",
|
|
'<p style="text-align: center;"><strong>Armor Protection</strong></p><ul><li>Dual-layer shockproof</li><li>Magnetic kickstand</li></ul>',
|
|
'<strong>Samsung Galaxy S24 Ultra Heavy Duty Armor Case</strong> | iFixKart',
|
|
'<p>Shop shockproof <strong>armor cover</strong> for Samsung Galaxy S24 Ultra.</p>',
|
|
'samsung case, armor cover, mobile accessories'
|
|
]
|
|
]
|
|
for row in products_data:
|
|
ws1.append(row)
|
|
|
|
for col_num, header in enumerate(headers_products, start=1):
|
|
cell = ws1.cell(row=1, column=col_num)
|
|
cell.fill = header_fill
|
|
cell.font = header_font
|
|
cell.alignment = Alignment(horizontal="center", vertical="center")
|
|
|
|
output = io.BytesIO()
|
|
wb.save(output)
|
|
output.seek(0)
|
|
|
|
return Response(
|
|
content=output.getvalue(),
|
|
media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
headers={"Content-Disposition": "attachment; filename=iFixKart_Bulk_Import_Demo_Template.xlsx"}
|
|
)
|
|
|
|
@router.get("/template/zip")
|
|
def download_zip_template():
|
|
"""
|
|
Generates sample Media ZIP archive template.
|
|
"""
|
|
output = io.BytesIO()
|
|
sample_image_bytes = (
|
|
b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01'
|
|
b'\x08\x02\x00\x00\x00\x90wS\xde\x00\x00\x00\x0cIDATx\x9cc\xf8\xcf\xc0'
|
|
b'\x00\x00\x03\x01\x01\x00\x18\xdd\x8d\xb0\x00\x00\x00\x00IEND\xaeB`\x82'
|
|
)
|
|
|
|
with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as zf:
|
|
zf.writestr("README.txt", "iFixKart Media ZIP Template")
|
|
zf.writestr("apple_iphone_15_pro_screenguard/1_front.png", sample_image_bytes)
|
|
zf.writestr("samsung_s24_ultra_case/1_main.png", sample_image_bytes)
|
|
|
|
output.seek(0)
|
|
return Response(
|
|
content=output.getvalue(),
|
|
media_type="application/zip",
|
|
headers={"Content-Disposition": "attachment; filename=iFixKart_Media_ZIP_Demo_Template.zip"}
|
|
)
|
|
|
|
from app.services.migration_engine.validation_engine import ValidationEngine
|
|
|
|
@router.post("/preview")
|
|
@router.get("/preview")
|
|
def preview_migration_data(job_id: Optional[str] = None, db: Session = Depends(get_db)):
|
|
"""
|
|
Dynamic preview endpoint returning mapped file columns and sample rows.
|
|
"""
|
|
job = None
|
|
if job_id:
|
|
job = db.query(MigrationJob).filter(MigrationJob.id == job_id).first()
|
|
if not job:
|
|
job = db.query(MigrationJob).order_by(MigrationJob.started_at.desc(), MigrationJob.id.desc()).first()
|
|
|
|
if not job:
|
|
return {"status": "success", "valid": True, "preview_items": []}
|
|
|
|
job_dir = StorageManager.get_job_dir(job.id)
|
|
dataset_dir = os.path.join(job_dir, "dataset")
|
|
if not os.path.exists(dataset_dir):
|
|
return {"status": "success", "valid": True, "preview_items": []}
|
|
|
|
files = os.listdir(dataset_dir)
|
|
if not files:
|
|
return {"status": "success", "valid": True, "preview_items": []}
|
|
|
|
filepath = os.path.join(dataset_dir, files[0])
|
|
fmt = os.path.splitext(filepath)[1].lower().replace(".", "").upper()
|
|
headers = DataFileParser.get_headers(filepath, fmt)
|
|
column_maps = ColumnMapper.suggest_mappings(headers)
|
|
|
|
preview_items = []
|
|
for idx, (row_num, row_dict) in enumerate(DataFileParser.stream_rows(filepath, fmt)):
|
|
if idx >= 10:
|
|
break
|
|
mapped = ColumnMapper.apply_mapping(row_dict, column_maps)
|
|
preview_items.append(mapped)
|
|
|
|
return {
|
|
"status": "success",
|
|
"job_id": job.id,
|
|
"valid": True,
|
|
"headers": headers,
|
|
"column_maps": column_maps,
|
|
"preview_items": preview_items
|
|
}
|
|
|
|
@router.get("/media-groups/preview")
|
|
@router.post("/media-groups/preview")
|
|
def get_media_groups_preview(db: Session = Depends(get_db)):
|
|
"""
|
|
Returns indexed MediaGroup records with their assets for the Media Library tab.
|
|
"""
|
|
groups = db.query(MediaGroup).order_by(MediaGroup.created_at.desc()).all()
|
|
res = []
|
|
for g in groups:
|
|
assets = []
|
|
for a in g.media_assets:
|
|
assets.append({
|
|
"id": a.id,
|
|
"original_filename": a.original_filename,
|
|
"cdn_url": a.cdn_url,
|
|
"thumbnail_url": a.thumbnail_url,
|
|
"file_size_bytes": a.file_size_bytes,
|
|
"mime_type": a.mime_type
|
|
})
|
|
res.append({
|
|
"id": g.id,
|
|
"media_key": g.media_key,
|
|
"brand_name": g.brand_name,
|
|
"model_name": g.model_name,
|
|
"variant_tag": g.variant_tag,
|
|
"assets_count": len(assets),
|
|
"media_assets": assets
|
|
})
|
|
|
|
return {
|
|
"status": "success",
|
|
"total_count": len(res),
|
|
"media_groups": res
|
|
}
|
|
|
|
@router.post("/dry-run")
|
|
@router.get("/dry-run")
|
|
@router.post("/dry_run")
|
|
@router.get("/dry_run")
|
|
def dry_run_migration_data(job_id: Optional[str] = None, db: Session = Depends(get_db)):
|
|
"""
|
|
Dynamic dry-run endpoint executing real ValidationEngine checks against uploaded file rows.
|
|
"""
|
|
job = None
|
|
if job_id:
|
|
job = db.query(MigrationJob).filter(MigrationJob.id == job_id).first()
|
|
if not job:
|
|
job = db.query(MigrationJob).order_by(MigrationJob.started_at.desc(), MigrationJob.id.desc()).first()
|
|
|
|
if not job:
|
|
return {
|
|
"status": "success",
|
|
"message": "No active migration job found to validate.",
|
|
"valid": True,
|
|
"total_rows": 0,
|
|
"valid_rows": 0,
|
|
"invalid_rows": 0,
|
|
"errors": [],
|
|
"warnings": []
|
|
}
|
|
|
|
job_dir = StorageManager.get_job_dir(job.id)
|
|
dataset_dir = os.path.join(job_dir, "dataset")
|
|
if not os.path.exists(dataset_dir) or not os.listdir(dataset_dir):
|
|
return {
|
|
"status": "success",
|
|
"message": f"Job {job.id} initialized. Dataset file pending.",
|
|
"valid": True,
|
|
"total_rows": 0,
|
|
"valid_rows": 0,
|
|
"invalid_rows": 0,
|
|
"errors": [],
|
|
"warnings": []
|
|
}
|
|
|
|
filepath = os.path.join(dataset_dir, os.listdir(dataset_dir)[0])
|
|
fmt = os.path.splitext(filepath)[1].lower().replace(".", "").upper()
|
|
headers = DataFileParser.get_headers(filepath, fmt)
|
|
column_maps = ColumnMapper.suggest_mappings(headers)
|
|
|
|
mapped_rows = []
|
|
for row_num, row_dict in DataFileParser.stream_rows(filepath, fmt):
|
|
mapped = ColumnMapper.apply_mapping(row_dict, column_maps)
|
|
mapped_rows.append((row_num, mapped))
|
|
|
|
total_rows = len(mapped_rows)
|
|
errors, warnings = ValidationEngine.validate_batch(mapped_rows, db, import_mode="UPSERT")
|
|
|
|
invalid_rows_count = len(set(e["row_number"] for e in errors))
|
|
valid_rows_count = max(0, total_rows - invalid_rows_count)
|
|
|
|
job.total_records = total_rows
|
|
db.commit()
|
|
|
|
return {
|
|
"status": "success",
|
|
"job_id": job.id,
|
|
"message": f"Dynamic Dry-Run Validation complete for {total_rows} rows.",
|
|
"valid": invalid_rows_count == 0,
|
|
"total_rows": total_rows,
|
|
"valid_rows": valid_rows_count,
|
|
"invalid_rows": invalid_rows_count,
|
|
"errors": errors[:100],
|
|
"warnings": warnings[:100],
|
|
"headers": headers,
|
|
"column_maps": column_maps
|
|
}
|
|
|
|
@router.post("/execute")
|
|
@router.get("/execute")
|
|
def execute_migration_batch(job_id: Optional[str] = None, db: Session = Depends(get_db)):
|
|
"""
|
|
Dynamic execution endpoint enqueuing job for persistent MigrationWorker processing.
|
|
"""
|
|
job = None
|
|
if job_id:
|
|
job = db.query(MigrationJob).filter(MigrationJob.id == job_id).first()
|
|
if not job:
|
|
job = db.query(MigrationJob).order_by(MigrationJob.started_at.desc(), MigrationJob.id.desc()).first()
|
|
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="No migration job found to execute.")
|
|
|
|
# Only mark as QUEUED if the job is not already RUNNING or COMPLETED
|
|
if job.status not in (JobStatusEnum.RUNNING, JobStatusEnum.COMPLETED):
|
|
job.status = JobStatusEnum.QUEUED
|
|
if job.current_phase == PhaseEnum.UPLOAD:
|
|
job.current_phase = PhaseEnum.VALIDATE
|
|
db.commit()
|
|
|
|
return {
|
|
"status": "success",
|
|
"job_id": job.id,
|
|
"batch_id": job.batch_id,
|
|
"job_status": job.status,
|
|
"message": f"Migration Job {job.id} enqueued. Background worker thread will execute pipeline phases."
|
|
}
|
|
|
|
|
|
|
|
@router.post("/upload")
|
|
async def upload_migration_file(
|
|
file: Optional[UploadFile] = File(None),
|
|
media_file: Optional[UploadFile] = File(None),
|
|
batch_type: str = Form("PRODUCTS"),
|
|
import_mode: str = Form("UPSERT"),
|
|
media_structure: str = Form("AUTO"),
|
|
user_id: str = Form("admin-user-01"),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""
|
|
64KB Streamed Upload Endpoint. Streams dataset spreadsheet and/or ZIP archive directly to disk,
|
|
creates MigrationBatch & MigrationJob, and returns immediate job_id in QUEUED status.
|
|
"""
|
|
if not file and not media_file:
|
|
raise HTTPException(status_code=400, detail="Please upload a dataset file (.xlsx/.csv) or media ZIP archive (.zip).")
|
|
|
|
batch_id = str(uuid.uuid4())
|
|
job_id = str(uuid.uuid4())
|
|
job_dir = StorageManager.get_job_dir(job_id)
|
|
|
|
# Save job configuration including media_structure (AUTO vs FLAT)
|
|
StorageManager.save_job_config(job_id, {"media_structure": media_structure, "import_mode": import_mode})
|
|
|
|
batch = MigrationBatch(
|
|
id=batch_id,
|
|
batch_type=BatchTypeEnum(batch_type) if batch_type in BatchTypeEnum.__members__ else BatchTypeEnum.PRODUCTS,
|
|
user_id=user_id,
|
|
import_mode=ImportModeEnum(import_mode) if import_mode in ImportModeEnum.__members__ else ImportModeEnum.UPSERT,
|
|
status=BatchStatusEnum.PENDING
|
|
)
|
|
db.add(batch)
|
|
|
|
file_name = ""
|
|
file_format = "CSV"
|
|
|
|
if file:
|
|
file_name = file.filename
|
|
ext = os.path.splitext(file.filename)[1].lower().replace(".", "")
|
|
file_format = ext.upper()
|
|
dataset_dest = os.path.join(job_dir, "dataset", file.filename)
|
|
await StorageManager.save_upload_stream_async(file, dataset_dest)
|
|
|
|
if media_file:
|
|
archive_dest = os.path.join(job_dir, "archives", media_file.filename)
|
|
await StorageManager.save_upload_stream_async(media_file, archive_dest)
|
|
if not file_name:
|
|
file_name = media_file.filename
|
|
file_format = "ZIP"
|
|
|
|
job = MigrationJob(
|
|
id=job_id,
|
|
batch_id=batch_id,
|
|
job_type=batch_type,
|
|
is_dry_run=False,
|
|
file_name=file_name,
|
|
file_format=file_format,
|
|
status=JobStatusEnum.QUEUED,
|
|
current_phase=PhaseEnum.UPLOAD
|
|
)
|
|
db.add(job)
|
|
db.commit()
|
|
|
|
return {
|
|
"status": "success",
|
|
"job_id": job.id,
|
|
"batch_id": batch.id,
|
|
"job_status": job.status,
|
|
"current_phase": job.current_phase,
|
|
"message": f"File uploaded safely to disk. Migration Job {job.id} queued for background processing."
|
|
}
|
|
|
|
@router.get("/batches")
|
|
@router.get("/jobs")
|
|
@router.get("/history")
|
|
def list_migration_jobs(db: Session = Depends(get_db)):
|
|
"""
|
|
List all historical migration jobs for the admin dashboard.
|
|
"""
|
|
jobs = db.query(MigrationJob).order_by(MigrationJob.started_at.desc(), MigrationJob.id.desc()).all()
|
|
res = []
|
|
for j in jobs:
|
|
dur_secs = 0
|
|
if j.started_at:
|
|
end_t = j.completed_at or j.finished_at or datetime.datetime.utcnow()
|
|
dur_secs = max(0, int((end_t - j.started_at).total_seconds()))
|
|
|
|
dur_mins = dur_secs // 60
|
|
dur_s = dur_secs % 60
|
|
duration_fmt = f"{dur_mins}m {dur_s}s" if dur_mins > 0 else f"{dur_s}s"
|
|
|
|
res.append({
|
|
"id": j.id,
|
|
"job_id": j.id,
|
|
"batch_id": j.batch_id,
|
|
"file_name": j.file_name,
|
|
"file_format": j.file_format,
|
|
"status": j.status,
|
|
"current_phase": j.current_phase,
|
|
"current_batch": j.current_batch,
|
|
"total_batches": j.total_batches,
|
|
"processed_records": j.processed_records,
|
|
"total_records": j.total_records,
|
|
"successful_records": j.successful_records,
|
|
"failed_records": j.failed_records,
|
|
"worker_id": j.worker_id,
|
|
"heartbeat_at": j.heartbeat_at.isoformat() if j.heartbeat_at else None,
|
|
"started_at": j.started_at.strftime("%Y-%m-%d %H:%M:%S") if j.started_at else "Pending",
|
|
"completed_at": j.completed_at.strftime("%Y-%m-%d %H:%M:%S") if j.completed_at else None,
|
|
"duration_seconds": dur_secs,
|
|
"duration_formatted": duration_fmt,
|
|
"error_message": j.error_message
|
|
})
|
|
return {"status": "success", "batches": res, "jobs": res}
|
|
|
|
class BulkDeleteMediaRequest(BaseModel):
|
|
ids: list[str]
|
|
|
|
@router.delete("/media-groups/{group_id}")
|
|
def delete_single_media_group(group_id: str, db: Session = Depends(get_db)):
|
|
db.query(MigrationMediaItem).filter(MigrationMediaItem.id == group_id).delete()
|
|
db.commit()
|
|
return {"status": "success", "message": f"Media item {group_id} deleted."}
|
|
|
|
@router.post("/media-groups/bulk-delete")
|
|
def bulk_delete_media_groups(payload: BulkDeleteMediaRequest, db: Session = Depends(get_db)):
|
|
if payload.ids:
|
|
db.query(MigrationMediaItem).filter(MigrationMediaItem.id.in_(payload.ids)).delete(synchronize_session=False)
|
|
db.commit()
|
|
return {"status": "success", "message": f"Deleted {len(payload.ids)} media items."}
|
|
|
|
@router.post("/purge-all")
|
|
@router.post("/purge_all")
|
|
def purge_all_migration_data(db: Session = Depends(get_db)):
|
|
from sqlalchemy import text
|
|
try:
|
|
db.execute(text("DELETE FROM migration_snapshots"))
|
|
db.execute(text("DELETE FROM migration_errors"))
|
|
db.execute(text("DELETE FROM migration_job_checkpoints"))
|
|
db.execute(text("DELETE FROM migration_media_items"))
|
|
db.execute(text("DELETE FROM migration_jobs"))
|
|
db.execute(text("DELETE FROM migration_batches"))
|
|
db.commit()
|
|
except Exception as e:
|
|
db.rollback()
|
|
print(f"Purge warning: {e}")
|
|
return {"status": "success", "message": "All migration history and media groups purged."}
|
|
|
|
@router.get("/jobs/{job_id}/status")
|
|
def get_job_telemetry(job_id: str, db: Session = Depends(get_db)):
|
|
"""
|
|
Fetches real-time telemetry metrics for a migration job.
|
|
"""
|
|
job = db.query(MigrationJob).filter(MigrationJob.id == job_id).first()
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="Migration job not found")
|
|
|
|
checkpoints = db.query(MigrationJobCheckpoint).filter(MigrationJobCheckpoint.job_id == job_id).all()
|
|
phase_checkpoints = {}
|
|
for c in checkpoints:
|
|
phase_checkpoints[c.phase] = {
|
|
"last_successful_batch": c.last_successful_batch,
|
|
"total_batches": c.total_batches,
|
|
"processed_records": c.processed_records,
|
|
"failed_records": c.failed_records
|
|
}
|
|
|
|
pct = 0.0
|
|
if job.total_records > 0:
|
|
pct = round((job.processed_records / job.total_records) * 100.0, 2)
|
|
|
|
elapsed_seconds = 0
|
|
if job.started_at:
|
|
end_time = job.completed_at or job.finished_at or datetime.datetime.utcnow()
|
|
elapsed_seconds = max(0, int((end_time - job.started_at).total_seconds()))
|
|
|
|
processing_rate = 0.0
|
|
estimated_seconds_remaining = None
|
|
if job.status == JobStatusEnum.QUEUED:
|
|
eta_formatted = "Queued in Line"
|
|
elif job.status == JobStatusEnum.RUNNING and job.processed_records == 0:
|
|
eta_formatted = "Starting Batch Processing..."
|
|
elif job.status == JobStatusEnum.COMPLETED:
|
|
eta_formatted = "0s (Completed)"
|
|
else:
|
|
eta_formatted = "Calculating..."
|
|
|
|
if elapsed_seconds > 0 and job.processed_records > 0:
|
|
processing_rate = round(job.processed_records / elapsed_seconds, 2)
|
|
remaining_records = max(0, job.total_records - job.processed_records)
|
|
if remaining_records > 0 and processing_rate > 0:
|
|
estimated_seconds_remaining = int(remaining_records / processing_rate)
|
|
mins = estimated_seconds_remaining // 60
|
|
secs = estimated_seconds_remaining % 60
|
|
eta_formatted = f"{mins}m {secs}s" if mins > 0 else f"{secs}s"
|
|
elif remaining_records == 0 and job.status == JobStatusEnum.COMPLETED:
|
|
eta_formatted = "0s (Completed)"
|
|
|
|
elapsed_mins = elapsed_seconds // 60
|
|
elapsed_secs = elapsed_seconds % 60
|
|
elapsed_formatted = f"{elapsed_mins}m {elapsed_secs}s" if elapsed_mins > 0 else f"{elapsed_secs}s"
|
|
|
|
return {
|
|
"status": "success",
|
|
"job_id": job.id,
|
|
"batch_id": job.batch_id,
|
|
"job_status": job.status,
|
|
"current_phase": job.current_phase,
|
|
"worker_id": job.worker_id,
|
|
"heartbeat_at": job.heartbeat_at.isoformat() if job.heartbeat_at else None,
|
|
"lease_version": job.lease_version,
|
|
"current_batch": job.current_batch,
|
|
"total_batches": job.total_batches,
|
|
"last_successful_batch": job.last_successful_batch,
|
|
"processed_records": job.processed_records,
|
|
"total_records": job.total_records,
|
|
"successful_records": job.successful_records,
|
|
"failed_records": job.failed_records,
|
|
"expected_products": job.expected_products,
|
|
"expected_variants": job.expected_variants,
|
|
"expected_media_items": job.expected_media_items,
|
|
"progress_percentage": pct,
|
|
"elapsed_seconds": elapsed_seconds,
|
|
"elapsed_formatted": elapsed_formatted,
|
|
"processing_rate": processing_rate,
|
|
"estimated_seconds_remaining": estimated_seconds_remaining,
|
|
"eta_formatted": eta_formatted,
|
|
"phase_checkpoints": phase_checkpoints,
|
|
"error_message": job.error_message
|
|
}
|
|
|
|
@router.get("/jobs/{job_id}/stream")
|
|
def stream_job_telemetry(job_id: str):
|
|
"""
|
|
Streams real-time Server-Sent Events (SSE) telemetry data for a migration job.
|
|
"""
|
|
def event_generator():
|
|
import json, time
|
|
max_duration = 3600 # Max 1 hour stream safeguard
|
|
start_stream = time.time()
|
|
|
|
while time.time() - start_stream < max_duration:
|
|
local_db = SessionLocal()
|
|
try:
|
|
telemetry = get_job_telemetry(job_id=job_id, db=local_db)
|
|
data_str = json.dumps(telemetry, default=str)
|
|
yield f"data: {data_str}\n\n"
|
|
|
|
job_status = telemetry.get("job_status")
|
|
if job_status in [JobStatusEnum.COMPLETED, JobStatusEnum.FAILED, JobStatusEnum.CANCELLED, "COMPLETED", "FAILED", "CANCELLED"]:
|
|
break
|
|
except Exception as e:
|
|
err_payload = json.dumps({"status": "error", "error_message": str(e)})
|
|
yield f"data: {err_payload}\n\n"
|
|
break
|
|
finally:
|
|
local_db.close()
|
|
|
|
time.sleep(1.0)
|
|
|
|
return StreamingResponse(event_generator(), media_type="text/event-stream")
|
|
|
|
@router.post("/jobs/{job_id}/cancel")
|
|
def cancel_job(job_id: str, db: Session = Depends(get_db)):
|
|
"""
|
|
Flags job status as CANCELLING. Worker finishes active batch transaction before halting cleanly.
|
|
"""
|
|
job = db.query(MigrationJob).filter(MigrationJob.id == job_id).first()
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="Migration job not found")
|
|
|
|
if job.status in (JobStatusEnum.COMPLETED, JobStatusEnum.CANCELLED, JobStatusEnum.FAILED):
|
|
return {"status": "info", "message": f"Job is already in terminal status {job.status}"}
|
|
|
|
job.status = JobStatusEnum.CANCELLING
|
|
job.cancel_requested_at = datetime.datetime.utcnow()
|
|
db.commit()
|
|
|
|
return {"status": "success", "message": f"Cancellation requested for job {job_id}. Worker will halt cleanly after current batch finishes."}
|
|
|
|
@router.post("/jobs/{job_id}/resume")
|
|
def resume_job(job_id: str, db: Session = Depends(get_db)):
|
|
"""
|
|
Resumes an interrupted, cancelled, or failed job from its last successful phase checkpoint.
|
|
"""
|
|
job = db.query(MigrationJob).filter(MigrationJob.id == job_id).first()
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="Migration job not found")
|
|
|
|
if job.status == JobStatusEnum.RUNNING:
|
|
return {"status": "info", "message": "Job is already running."}
|
|
|
|
job.status = JobStatusEnum.QUEUED
|
|
job.cancel_requested_at = None
|
|
job.error_message = None
|
|
db.commit()
|
|
|
|
return {"status": "success", "message": f"Job {job_id} re-queued and will resume from phase {job.current_phase} batch {job.last_successful_batch}."}
|
|
|
|
@router.get("/jobs/{job_id}/errors/export")
|
|
def export_job_errors(job_id: str, db: Session = Depends(get_db)):
|
|
"""
|
|
Exports CSV failure report: Row Number, SKU, Product Name, File Name, Phase, Error Type, Error Message, Retry Status.
|
|
"""
|
|
job = db.query(MigrationJob).filter(MigrationJob.id == job_id).first()
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="Migration job not found")
|
|
|
|
errors = db.query(MigrationError).filter(MigrationError.job_id == job_id).order_by(MigrationError.row_number.asc()).all()
|
|
|
|
output = io.StringIO()
|
|
writer = csv.writer(output)
|
|
writer.writerow([
|
|
"Row Number", "SKU", "Product Name", "File Name", "Entity Type",
|
|
"Phase", "Error Type", "Severity", "Error Message", "Suggested Fix", "Retry Status"
|
|
])
|
|
|
|
for e in errors:
|
|
writer.writerow([
|
|
e.row_number, e.sku or "", e.product_name or "", e.file_name or "", e.entity_type or "",
|
|
e.phase, e.error_type, e.severity, e.error_message, e.suggested_fix or "", e.retry_status
|
|
])
|
|
|
|
output.seek(0)
|
|
return StreamingResponse(
|
|
io.BytesIO(output.getvalue().encode("utf-8")),
|
|
media_type="text/csv",
|
|
headers={"Content-Disposition": f"attachment; filename=Migration_Errors_Job_{job_id[:8]}.csv"}
|
|
)
|
|
|
|
@router.get("/jobs/{job_id}/media")
|
|
def list_job_media(
|
|
job_id: str,
|
|
page: int = 1,
|
|
limit: int = 50,
|
|
search: Optional[str] = None,
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""
|
|
Server-side paginated media gallery for admin console to view 30,000+ indexed media assets without DOM lag.
|
|
"""
|
|
query = db.query(MigrationMediaItem).filter(MigrationMediaItem.job_id == job_id)
|
|
if search:
|
|
query = query.filter(
|
|
(MigrationMediaItem.file_name.ilike(f"%{search}%")) |
|
|
(MigrationMediaItem.media_key.ilike(f"%{search}%")) |
|
|
(MigrationMediaItem.sha256.ilike(f"%{search}%"))
|
|
)
|
|
|
|
total_count = query.count()
|
|
items = query.offset((page - 1) * limit).limit(limit).all()
|
|
|
|
res = []
|
|
for item in items:
|
|
res.append({
|
|
"id": item.id,
|
|
"batch_number": item.batch_number,
|
|
"file_name": item.file_name,
|
|
"archive_name": item.archive_name,
|
|
"zip_entry_path": item.zip_entry_path,
|
|
"media_key": item.media_key,
|
|
"sha256": item.sha256,
|
|
"storage_path": item.storage_path,
|
|
"cdn_url": f"/uploads/migrations/{job_id}/media/{item.sha256[:2]}/{item.sha256}.jpg" if item.sha256 else None,
|
|
"status": item.status,
|
|
"error": item.error
|
|
})
|
|
|
|
return {
|
|
"status": "success",
|
|
"total_count": total_count,
|
|
"page": page,
|
|
"limit": limit,
|
|
"total_pages": (total_count // limit) + (1 if total_count % limit > 0 else 0),
|
|
"items": res
|
|
}
|