fix(migration): support full MediaGroup/MediaAsset cascade delete, disk cleanup, unlinking, and rollback
This commit is contained in:
parent
3d7c2dec0a
commit
211666347e
1 changed files with 177 additions and 9 deletions
|
|
@ -10,6 +10,7 @@ from typing import List, Dict, Any, Optional
|
||||||
from fastapi import APIRouter, UploadFile, File, Form, Depends, HTTPException, status
|
from fastapi import APIRouter, UploadFile, File, Form, Depends, HTTPException, status
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from fastapi.responses import Response, StreamingResponse
|
from fastapi.responses import Response, StreamingResponse
|
||||||
|
from sqlalchemy import or_, text
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
from app.core.database.db_session import get_db, SessionLocal
|
from app.core.database.db_session import get_db, SessionLocal
|
||||||
import openpyxl
|
import openpyxl
|
||||||
|
|
@ -489,37 +490,204 @@ def list_migration_jobs(db: Session = Depends(get_db)):
|
||||||
return {"status": "success", "batches": res, "jobs": res}
|
return {"status": "success", "batches": res, "jobs": res}
|
||||||
|
|
||||||
class BulkDeleteMediaRequest(BaseModel):
|
class BulkDeleteMediaRequest(BaseModel):
|
||||||
ids: list[str]
|
ids: Optional[list[str]] = None
|
||||||
|
group_ids: Optional[list[str]] = None
|
||||||
|
|
||||||
|
def target_ids(self) -> list[str]:
|
||||||
|
combined = []
|
||||||
|
if self.ids:
|
||||||
|
combined.extend(self.ids)
|
||||||
|
if self.group_ids:
|
||||||
|
combined.extend(self.group_ids)
|
||||||
|
return list(dict.fromkeys(str(x).strip() for x in combined if x and str(x).strip()))
|
||||||
|
|
||||||
@router.delete("/media-groups/{group_id}")
|
@router.delete("/media-groups/{group_id}")
|
||||||
def delete_single_media_group(group_id: str, db: Session = Depends(get_db)):
|
def delete_single_media_group(group_id: str, db: Session = Depends(get_db)):
|
||||||
db.query(MigrationMediaItem).filter(MigrationMediaItem.id == group_id).delete()
|
"""
|
||||||
|
Deletes a MediaGroup along with its MediaAssets, linked variant/product images, and disk files.
|
||||||
|
Accepts MediaGroup id, media_key, or staging MigrationMediaItem id.
|
||||||
|
"""
|
||||||
|
# 1. Search for MediaGroup by id or media_key
|
||||||
|
mg = db.query(MediaGroup).filter(or_(MediaGroup.id == group_id, MediaGroup.media_key == group_id)).first()
|
||||||
|
|
||||||
|
# 2. If not found, check if group_id is a MigrationMediaItem id
|
||||||
|
if not mg:
|
||||||
|
item = db.query(MigrationMediaItem).filter(MigrationMediaItem.id == group_id).first()
|
||||||
|
if item and item.media_key:
|
||||||
|
mg = db.query(MediaGroup).filter(MediaGroup.media_key == item.media_key).first()
|
||||||
|
try:
|
||||||
|
db.delete(item)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if mg:
|
||||||
|
media_key = mg.media_key
|
||||||
|
# Delete physical files and collect URLs to unlink
|
||||||
|
asset_urls = []
|
||||||
|
for asset in (mg.media_assets or []):
|
||||||
|
if asset.storage_path and os.path.exists(asset.storage_path):
|
||||||
|
try:
|
||||||
|
os.remove(asset.storage_path)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
if asset.cdn_url:
|
||||||
|
asset_urls.append(asset.cdn_url)
|
||||||
|
|
||||||
|
# Unlink from VariantImage and ProductImage
|
||||||
|
if asset_urls:
|
||||||
|
db.query(VariantImage).filter(VariantImage.image_url.in_(asset_urls)).delete(synchronize_session=False)
|
||||||
|
db.query(ProductImage).filter(ProductImage.image_url.in_(asset_urls)).delete(synchronize_session=False)
|
||||||
|
|
||||||
|
# Delete any staging media items matching this key
|
||||||
|
if media_key:
|
||||||
|
db.query(MigrationMediaItem).filter(
|
||||||
|
or_(MigrationMediaItem.media_key == media_key, MigrationMediaItem.id == group_id)
|
||||||
|
).delete(synchronize_session=False)
|
||||||
|
|
||||||
|
# Delete media group (cascades to MediaAsset)
|
||||||
|
db.delete(mg)
|
||||||
db.commit()
|
db.commit()
|
||||||
|
return {"status": "success", "message": f"Media item {media_key} deleted successfully."}
|
||||||
|
|
||||||
|
# Fallback: check if staging items exist
|
||||||
|
deleted_items = db.query(MigrationMediaItem).filter(
|
||||||
|
or_(MigrationMediaItem.id == group_id, MigrationMediaItem.media_key == group_id)
|
||||||
|
).delete(synchronize_session=False)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
if deleted_items > 0:
|
||||||
return {"status": "success", "message": f"Media item {group_id} deleted."}
|
return {"status": "success", "message": f"Media item {group_id} deleted."}
|
||||||
|
|
||||||
|
return {"status": "success", "message": f"Media item {group_id} removed."}
|
||||||
|
|
||||||
@router.post("/media-groups/bulk-delete")
|
@router.post("/media-groups/bulk-delete")
|
||||||
def bulk_delete_media_groups(payload: BulkDeleteMediaRequest, db: Session = Depends(get_db)):
|
def bulk_delete_media_groups(payload: BulkDeleteMediaRequest, db: Session = Depends(get_db)):
|
||||||
if payload.ids:
|
"""
|
||||||
db.query(MigrationMediaItem).filter(MigrationMediaItem.id.in_(payload.ids)).delete(synchronize_session=False)
|
Bulk deletes multiple MediaGroup records, their MediaAssets, linked variant/product images, and disk files.
|
||||||
|
Supports both payload.ids and payload.group_ids.
|
||||||
|
"""
|
||||||
|
target_ids = payload.target_ids()
|
||||||
|
if not target_ids:
|
||||||
|
return {"status": "success", "message": "No media keys specified for deletion.", "deleted_groups_count": 0}
|
||||||
|
|
||||||
|
# Find all MediaGroups matching IDs or media_keys
|
||||||
|
groups = db.query(MediaGroup).filter(
|
||||||
|
or_(MediaGroup.id.in_(target_ids), MediaGroup.media_key.in_(target_ids))
|
||||||
|
).all()
|
||||||
|
media_keys = [g.media_key for g in groups if g.media_key]
|
||||||
|
|
||||||
|
# Also check if any target_ids are MigrationMediaItem IDs
|
||||||
|
staging_items = db.query(MigrationMediaItem).filter(MigrationMediaItem.id.in_(target_ids)).all()
|
||||||
|
staging_keys = [i.media_key for i in staging_items if i.media_key]
|
||||||
|
if staging_keys:
|
||||||
|
more_groups = db.query(MediaGroup).filter(MediaGroup.media_key.in_(staging_keys)).all()
|
||||||
|
for mg in more_groups:
|
||||||
|
if mg not in groups:
|
||||||
|
groups.append(mg)
|
||||||
|
if mg.media_key:
|
||||||
|
media_keys.append(mg.media_key)
|
||||||
|
|
||||||
|
# Clean up physical files and collect asset URLs
|
||||||
|
all_cdn_urls = []
|
||||||
|
for g in groups:
|
||||||
|
for asset in (g.media_assets or []):
|
||||||
|
if asset.storage_path and os.path.exists(asset.storage_path):
|
||||||
|
try:
|
||||||
|
os.remove(asset.storage_path)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
if asset.cdn_url:
|
||||||
|
all_cdn_urls.append(asset.cdn_url)
|
||||||
|
db.delete(g)
|
||||||
|
|
||||||
|
# Unlink images from variants and products
|
||||||
|
if all_cdn_urls:
|
||||||
|
db.query(VariantImage).filter(VariantImage.image_url.in_(all_cdn_urls)).delete(synchronize_session=False)
|
||||||
|
db.query(ProductImage).filter(ProductImage.image_url.in_(all_cdn_urls)).delete(synchronize_session=False)
|
||||||
|
|
||||||
|
# Clean up staging items
|
||||||
|
all_clean_keys = list(set(target_ids + media_keys + staging_keys))
|
||||||
|
db.query(MigrationMediaItem).filter(
|
||||||
|
or_(MigrationMediaItem.id.in_(all_clean_keys), MigrationMediaItem.media_key.in_(all_clean_keys))
|
||||||
|
).delete(synchronize_session=False)
|
||||||
|
|
||||||
db.commit()
|
db.commit()
|
||||||
return {"status": "success", "message": f"Deleted {len(payload.ids)} media items."}
|
count = len(groups)
|
||||||
|
return {"status": "success", "message": f"Deleted {count} media key(s).", "deleted_groups_count": count}
|
||||||
|
|
||||||
|
@router.post("/rollback/{batch_id}")
|
||||||
|
def rollback_migration_batch(batch_id: str, db: Session = Depends(get_db)):
|
||||||
|
"""
|
||||||
|
Reverts inserted and updated records for a migration batch using migration_snapshots.
|
||||||
|
"""
|
||||||
|
batch = db.query(MigrationBatch).filter(MigrationBatch.id == batch_id).first()
|
||||||
|
if not batch:
|
||||||
|
raise HTTPException(status_code=404, detail="Migration batch not found")
|
||||||
|
|
||||||
|
snapshots = db.query(MigrationSnapshot).filter(
|
||||||
|
MigrationSnapshot.batch_id == batch_id
|
||||||
|
).order_by(MigrationSnapshot.created_at.desc()).all()
|
||||||
|
|
||||||
|
reverted_inserts = 0
|
||||||
|
reverted_updates = 0
|
||||||
|
|
||||||
|
for snap in snapshots:
|
||||||
|
if snap.entity_table == "product_variants":
|
||||||
|
variant = db.query(ProductVariant).filter(ProductVariant.variant_id == snap.entity_id).first()
|
||||||
|
if not variant:
|
||||||
|
continue
|
||||||
|
if snap.action_taken == "INSERTED":
|
||||||
|
# Delete variant attributes & images
|
||||||
|
db.query(VariantAttribute).filter(VariantAttribute.variant_id == variant.variant_id).delete(synchronize_session=False)
|
||||||
|
db.query(VariantImage).filter(VariantImage.variant_id == variant.variant_id).delete(synchronize_session=False)
|
||||||
|
|
||||||
|
prod_id = variant.product_id
|
||||||
|
db.delete(variant)
|
||||||
|
reverted_inserts += 1
|
||||||
|
|
||||||
|
# Clean up empty parent product
|
||||||
|
if prod_id:
|
||||||
|
remaining = db.query(ProductVariant).filter(ProductVariant.product_id == prod_id).count()
|
||||||
|
if remaining <= 1:
|
||||||
|
prod = db.query(Product).filter(Product.product_id == prod_id).first()
|
||||||
|
if prod:
|
||||||
|
db.query(ProductImage).filter(ProductImage.product_id == prod_id).delete(synchronize_session=False)
|
||||||
|
db.delete(prod)
|
||||||
|
|
||||||
|
elif snap.action_taken == "UPDATED" and snap.previous_state:
|
||||||
|
prev = snap.previous_state
|
||||||
|
if "price" in prev:
|
||||||
|
variant.price = prev["price"]
|
||||||
|
if "cost_price" in prev:
|
||||||
|
variant.cost_price = prev["cost_price"]
|
||||||
|
if "sku" in prev:
|
||||||
|
variant.sku = prev["sku"]
|
||||||
|
reverted_updates += 1
|
||||||
|
|
||||||
|
batch.status = BatchStatusEnum.ROLLED_BACK
|
||||||
|
db.commit()
|
||||||
|
return {
|
||||||
|
"status": "success",
|
||||||
|
"message": f"Rollback complete: {reverted_inserts} inserted variants removed, {reverted_updates} updated variants restored."
|
||||||
|
}
|
||||||
|
|
||||||
@router.post("/purge-all")
|
@router.post("/purge-all")
|
||||||
@router.post("/purge_all")
|
@router.post("/purge_all")
|
||||||
def purge_all_migration_data(db: Session = Depends(get_db)):
|
def purge_all_migration_data(db: Session = Depends(get_db)):
|
||||||
from sqlalchemy import text
|
|
||||||
try:
|
try:
|
||||||
db.execute(text("DELETE FROM migration_snapshots"))
|
db.execute(text("DELETE FROM migration_snapshots"))
|
||||||
db.execute(text("DELETE FROM migration_errors"))
|
db.execute(text("DELETE FROM migration_errors"))
|
||||||
db.execute(text("DELETE FROM migration_job_checkpoints"))
|
db.execute(text("DELETE FROM migration_job_checkpoints"))
|
||||||
db.execute(text("DELETE FROM migration_media_items"))
|
db.execute(text("DELETE FROM migration_media_items"))
|
||||||
|
db.execute(text("DELETE FROM media_library"))
|
||||||
|
db.execute(text("DELETE FROM media_groups"))
|
||||||
db.execute(text("DELETE FROM migration_jobs"))
|
db.execute(text("DELETE FROM migration_jobs"))
|
||||||
db.execute(text("DELETE FROM migration_batches"))
|
db.execute(text("DELETE FROM migration_batches"))
|
||||||
db.commit()
|
db.commit()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
print(f"Purge warning: {e}")
|
print(f"Purge warning: {e}")
|
||||||
return {"status": "success", "message": "All migration history and media groups purged."}
|
return {"status": "success", "message": "All migration history, jobs, and media groups purged."}
|
||||||
|
|
||||||
@router.get("/jobs/{job_id}/status")
|
@router.get("/jobs/{job_id}/status")
|
||||||
def get_job_telemetry(job_id: str, db: Session = Depends(get_db)):
|
def get_job_telemetry(job_id: str, db: Session = Depends(get_db)):
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue