497 lines
24 KiB
Python
497 lines
24 KiB
Python
|
|
# pyrefly: ignore [missing-import]
|
|
import ulid
|
|
import re
|
|
from typing import List, Dict, Any, Tuple
|
|
from sqlalchemy.orm import Session
|
|
import app.models.db_base
|
|
from app.models.ProductModel import Product, ProductVariant, VariantImage, AttributeType, VariantAttribute
|
|
from app.models.BrandModel import Brand
|
|
from app.models.CategoryModel import Category
|
|
from app.models.DeviceCatalogModel import DeviceSeries, DeviceModel
|
|
from app.models.MigrationModel import MediaGroup, MediaAsset, MigrationSnapshot
|
|
from app.services.migration_engine.media_resolver import MediaResolver
|
|
|
|
BRAND_LOGOS = {}
|
|
|
|
CATEGORY_IMAGES = {}
|
|
|
|
class UpsertExecutor:
|
|
"""
|
|
Executes chunked batch transactions according to selected Import Mode:
|
|
- CREATE_ONLY
|
|
- UPDATE_EXISTING
|
|
- UPSERT
|
|
- SKIP_EXISTING
|
|
|
|
Saves delta snapshots in migration_snapshots for atomic Rollback.
|
|
Groups multiple variants under single parent Product.
|
|
Populates Master Attributes (attribute_types) and Variant Attributes (variant_attributes).
|
|
Attaches variant images via Media Keys.
|
|
"""
|
|
|
|
@classmethod
|
|
def execute_batch(
|
|
cls,
|
|
batch_id: str,
|
|
mapped_rows: List[Tuple[int, Dict[str, Any]]],
|
|
db_session: Session,
|
|
import_mode: str = "UPSERT",
|
|
brand_cache: dict = None,
|
|
category_cache: dict = None,
|
|
series_cache: dict = None,
|
|
model_cache: dict = None,
|
|
product_cache: dict = None,
|
|
attr_type_cache: dict = None,
|
|
media_group_cache: dict = None
|
|
) -> Tuple[int, int, List[Dict[str, Any]]]:
|
|
"""
|
|
Executes database writes for a list of (row_number, mapped_row) items.
|
|
Returns (success_count, fail_count, list_of_runtime_errors).
|
|
"""
|
|
success_count = 0
|
|
fail_count = 0
|
|
runtime_errors = []
|
|
|
|
for row_number, mapped_row in mapped_rows:
|
|
sku = mapped_row.get("sku")
|
|
if not sku:
|
|
fail_count += 1
|
|
runtime_errors.append({
|
|
"row_number": row_number,
|
|
"sku": None,
|
|
"severity": "ERROR",
|
|
"field_name": "sku",
|
|
"error_message": "Missing SKU in execution batch",
|
|
"suggested_fix": "Ensure row contains a valid SKU"
|
|
})
|
|
continue
|
|
|
|
sp = None
|
|
try:
|
|
sp = db_session.begin_nested()
|
|
# 0. Get device type
|
|
device_type_val = mapped_row.get("device_type") or (mapped_row.get("attributes") or {}).get("device_type")
|
|
if device_type_val:
|
|
device_type_val = str(device_type_val).strip().lower()
|
|
if device_type_val not in ("laptop", "tablet", "mobile"):
|
|
device_type_val = None
|
|
|
|
# 1. Resolve Brand & Category
|
|
brand_name = mapped_row.get("brand")
|
|
brand_obj = None
|
|
if brand_name:
|
|
if brand_cache is not None and brand_name in brand_cache:
|
|
brand_obj = brand_cache[brand_name]
|
|
else:
|
|
brand_obj = db_session.query(Brand).filter(Brand.name == brand_name).first()
|
|
if not brand_obj:
|
|
b_id = str(ulid.ULID())
|
|
brand_obj = Brand(
|
|
brand_id=b_id,
|
|
name=brand_name,
|
|
slug=brand_name.lower().replace(" ", "-"),
|
|
logo_url=BRAND_LOGOS.get(brand_name),
|
|
is_active=True
|
|
)
|
|
db_session.add(brand_obj)
|
|
db_session.flush()
|
|
if brand_cache is not None:
|
|
brand_cache[brand_name] = brand_obj
|
|
|
|
# Map brand device type dynamically
|
|
if brand_obj and device_type_val:
|
|
from app.models.BrandDeviceTypeModel import BrandDeviceType
|
|
exists_bdt = db_session.query(BrandDeviceType).filter(
|
|
BrandDeviceType.brand_id == brand_obj.brand_id,
|
|
BrandDeviceType.device_type == device_type_val
|
|
).first()
|
|
if not exists_bdt:
|
|
bdt = BrandDeviceType(
|
|
brand_id=brand_obj.brand_id,
|
|
device_type=device_type_val
|
|
)
|
|
db_session.add(bdt)
|
|
|
|
cat_name = mapped_row.get("category") or "General"
|
|
parent_cat_name = mapped_row.get("parent_category")
|
|
is_parent_feature_val = mapped_row.get("is_parent_feature")
|
|
is_parent_feature_bool = False
|
|
if is_parent_feature_val:
|
|
is_parent_feature_bool = str(is_parent_feature_val).strip().lower() in ("true", "1", "yes")
|
|
|
|
cat_obj = None
|
|
if category_cache is not None and cat_name in category_cache:
|
|
cat_obj = category_cache[cat_name]
|
|
else:
|
|
cat_obj = db_session.query(Category).filter(Category.name == cat_name).first()
|
|
if not cat_obj:
|
|
c_id = str(ulid.ULID())
|
|
cat_obj = Category(
|
|
category_id=c_id,
|
|
name=cat_name,
|
|
slug=cat_name.lower().replace(" ", "-"),
|
|
image_url=CATEGORY_IMAGES.get(cat_name),
|
|
is_parent_feature=is_parent_feature_bool,
|
|
is_active=True
|
|
)
|
|
db_session.add(cat_obj)
|
|
db_session.flush()
|
|
if category_cache is not None:
|
|
category_cache[cat_name] = cat_obj
|
|
|
|
# Update Category properties if needed
|
|
if cat_obj:
|
|
if is_parent_feature_bool and not cat_obj.is_parent_feature:
|
|
cat_obj.is_parent_feature = True
|
|
|
|
if parent_cat_name:
|
|
parent_cat_name = str(parent_cat_name).strip()
|
|
parent_cat = None
|
|
if category_cache is not None and parent_cat_name in category_cache:
|
|
parent_cat = category_cache[parent_cat_name]
|
|
else:
|
|
parent_cat = db_session.query(Category).filter(Category.name == parent_cat_name).first()
|
|
if not parent_cat:
|
|
pc_id = str(ulid.ULID())
|
|
parent_cat = Category(
|
|
category_id=pc_id,
|
|
name=parent_cat_name,
|
|
slug=parent_cat_name.lower().replace(" ", "-"),
|
|
image_url=CATEGORY_IMAGES.get(parent_cat_name),
|
|
is_active=True
|
|
)
|
|
db_session.add(parent_cat)
|
|
db_session.flush()
|
|
if category_cache is not None:
|
|
category_cache[parent_cat_name] = parent_cat
|
|
|
|
if parent_cat and cat_obj.parent_category_id != parent_cat.category_id:
|
|
cat_obj.parent_category_id = parent_cat.category_id
|
|
|
|
# 1b. Resolve Global Device Series & Device Model
|
|
device_model_val = (
|
|
mapped_row.get("device_model")
|
|
or mapped_row.get("model")
|
|
or (mapped_row.get("attributes") or {}).get("Device Model")
|
|
or (mapped_row.get("attributes") or {}).get("device_model")
|
|
or (mapped_row.get("attributes") or {}).get("Model")
|
|
)
|
|
device_series_val = (
|
|
mapped_row.get("device_series")
|
|
or mapped_row.get("series")
|
|
or (mapped_row.get("attributes") or {}).get("Device Series")
|
|
or (mapped_row.get("attributes") or {}).get("device_series")
|
|
or (mapped_row.get("attributes") or {}).get("Series")
|
|
)
|
|
|
|
series_obj = None
|
|
model_obj = None
|
|
|
|
if brand_obj:
|
|
# Resolve Device Series
|
|
s_name = device_series_val or f"{brand_obj.name} Series"
|
|
s_key = f"{brand_obj.brand_id}:{s_name}"
|
|
if series_cache is not None and s_key in series_cache:
|
|
series_obj = series_cache[s_key]
|
|
else:
|
|
s_slug = s_name.lower().replace(" ", "-").replace("/", "-")
|
|
series_obj = db_session.query(DeviceSeries).filter(
|
|
DeviceSeries.brand_id == brand_obj.brand_id,
|
|
DeviceSeries.name == s_name
|
|
).first()
|
|
if not series_obj:
|
|
series_obj = db_session.query(DeviceSeries).filter(
|
|
DeviceSeries.slug == s_slug
|
|
).first()
|
|
if not series_obj:
|
|
s_id = str(ulid.ULID())
|
|
series_obj = DeviceSeries(
|
|
series_id=s_id,
|
|
brand_id=brand_obj.brand_id,
|
|
name=s_name,
|
|
slug=s_slug,
|
|
device_type=device_type_val,
|
|
is_active=True
|
|
)
|
|
db_session.add(series_obj)
|
|
db_session.flush()
|
|
if series_cache is not None:
|
|
series_cache[s_key] = series_obj
|
|
|
|
if series_obj and device_type_val and not series_obj.device_type:
|
|
series_obj.device_type = device_type_val
|
|
|
|
# Resolve Device Model
|
|
if device_model_val:
|
|
m_name = str(device_model_val).strip()
|
|
m_key = f"{brand_obj.brand_id}:{m_name}"
|
|
if model_cache is not None and m_key in model_cache:
|
|
model_obj = model_cache[m_key]
|
|
else:
|
|
m_slug = f"{brand_obj.slug}-{m_name.lower().replace(' ', '-').replace('/', '-')}"
|
|
model_obj = db_session.query(DeviceModel).filter(
|
|
DeviceModel.brand_id == brand_obj.brand_id,
|
|
DeviceModel.name == m_name
|
|
).first()
|
|
if not model_obj:
|
|
model_obj = db_session.query(DeviceModel).filter(
|
|
DeviceModel.slug == m_slug
|
|
).first()
|
|
if not model_obj:
|
|
m_id = str(ulid.ULID())
|
|
model_obj = DeviceModel(
|
|
model_id=m_id,
|
|
series_id=series_obj.series_id,
|
|
brand_id=brand_obj.brand_id,
|
|
name=m_name,
|
|
slug=m_slug,
|
|
device_type=device_type_val,
|
|
full_path=f"/{brand_obj.slug}/{series_obj.slug}/{m_slug}",
|
|
is_active=True
|
|
)
|
|
db_session.add(model_obj)
|
|
db_session.flush()
|
|
if model_cache is not None:
|
|
model_cache[m_key] = model_obj
|
|
|
|
if model_obj and device_type_val and not model_obj.device_type:
|
|
model_obj.device_type = device_type_val
|
|
|
|
# 2. Query Existing Variant
|
|
variant = db_session.query(ProductVariant).filter(ProductVariant.sku == sku).first()
|
|
|
|
if variant and import_mode == "SKIP_EXISTING":
|
|
sp.rollback()
|
|
continue
|
|
|
|
if not variant and import_mode == "UPDATE_EXISTING":
|
|
sp.rollback()
|
|
fail_count += 1
|
|
runtime_errors.append({
|
|
"row_number": row_number,
|
|
"sku": sku,
|
|
"severity": "WARNING",
|
|
"field_name": "sku",
|
|
"error_message": f"Skipped SKU '{sku}' - does not exist",
|
|
"suggested_fix": "Set mode to UPSERT to insert new items"
|
|
})
|
|
continue
|
|
|
|
if variant and import_mode == "CREATE_ONLY":
|
|
sp.rollback()
|
|
fail_count += 1
|
|
runtime_errors.append({
|
|
"row_number": row_number,
|
|
"sku": sku,
|
|
"severity": "ERROR",
|
|
"field_name": "sku",
|
|
"error_message": f"Cannot create SKU '{sku}' - already exists",
|
|
"suggested_fix": "Set mode to UPSERT or UPDATE_EXISTING"
|
|
})
|
|
continue
|
|
|
|
# 3. Resolve Parent Product (Group multiple variants under single parent Product)
|
|
price_val = float(mapped_row.get("price", 0.0) or 0.0)
|
|
raw_name = mapped_row.get("name") or f"Product {sku}"
|
|
|
|
# Check for explicit parent_name or clean name by stripping trailing variant tags like '(Black - Matte)'
|
|
parent_title = mapped_row.get("parent_name")
|
|
if not parent_title:
|
|
parent_title = re.sub(r'\s*\([^)]*\)$', '', raw_name).strip()
|
|
|
|
parent_slug = parent_title.lower().replace(" ", "-")
|
|
|
|
# Find or Create Parent Product
|
|
prod = None
|
|
if product_cache is not None and parent_slug in product_cache:
|
|
prod = product_cache[parent_slug]
|
|
else:
|
|
prod = db_session.query(Product).filter(Product.slug == parent_slug).first()
|
|
if not prod:
|
|
prod = db_session.query(Product).filter(Product.name == parent_title).first()
|
|
|
|
if not prod:
|
|
p_id = str(ulid.ULID())
|
|
prod = Product(
|
|
product_id=p_id,
|
|
name=parent_title,
|
|
slug=parent_slug,
|
|
full_path=f"/{cat_obj.slug}/{parent_slug}",
|
|
brand_id=brand_obj.brand_id if brand_obj else None,
|
|
device_series_id=series_obj.series_id if series_obj else None,
|
|
device_model_id=model_obj.model_id if model_obj else None,
|
|
category_id=cat_obj.category_id,
|
|
device_type=device_type_val,
|
|
status="active"
|
|
)
|
|
db_session.add(prod)
|
|
db_session.flush()
|
|
else:
|
|
if series_obj and not prod.device_series_id:
|
|
prod.device_series_id = series_obj.series_id
|
|
if model_obj and not prod.device_model_id:
|
|
prod.device_model_id = model_obj.model_id
|
|
if device_type_val and not prod.device_type:
|
|
prod.device_type = device_type_val
|
|
if product_cache is not None:
|
|
product_cache[parent_slug] = prod
|
|
|
|
# 4. Create or Update Variant
|
|
if not variant:
|
|
v_id = str(ulid.ULID())
|
|
variant = ProductVariant(
|
|
variant_id=v_id,
|
|
product_id=prod.product_id,
|
|
sku=sku,
|
|
price=price_val,
|
|
cost_price=price_val * 0.7,
|
|
status="active"
|
|
)
|
|
db_session.add(variant)
|
|
|
|
snapshot = MigrationSnapshot(
|
|
batch_id=batch_id,
|
|
entity_table="product_variants",
|
|
entity_id=variant.variant_id,
|
|
action_taken="INSERTED",
|
|
previous_state=None
|
|
)
|
|
db_session.add(snapshot)
|
|
|
|
else:
|
|
prev_state = {
|
|
"price": float(variant.price or 0.0),
|
|
"sku": str(variant.sku)
|
|
}
|
|
variant.price = price_val
|
|
variant.product_id = prod.product_id
|
|
|
|
snapshot = MigrationSnapshot(
|
|
batch_id=batch_id,
|
|
entity_table="product_variants",
|
|
entity_id=variant.variant_id,
|
|
action_taken="UPDATED",
|
|
previous_state=prev_state
|
|
)
|
|
db_session.add(snapshot)
|
|
|
|
# 5. Dynamic Master Attributes (attribute_types) & Variant Attributes (variant_attributes)
|
|
# Collect attributes from mapped_row.get("attributes") or key-values in row
|
|
all_attributes: Dict[str, Any] = {}
|
|
if "attributes" in mapped_row and isinstance(mapped_row["attributes"], dict):
|
|
all_attributes.update(mapped_row["attributes"])
|
|
|
|
for key, val in mapped_row.items():
|
|
if key not in ("sku", "name", "parent_name", "price", "stock", "brand", "category", "barcode", "description", "attributes"):
|
|
if val:
|
|
all_attributes[key] = val
|
|
|
|
if all_attributes:
|
|
# Clear previous variant_attributes links for this variant
|
|
db_session.query(VariantAttribute).filter(VariantAttribute.variant_id == variant.variant_id).delete()
|
|
|
|
for attr_name, attr_val in all_attributes.items():
|
|
if not attr_val:
|
|
continue
|
|
|
|
attr_code = attr_name.lower().strip().replace(" ", "_").replace("-", "_")
|
|
|
|
# Find or Create AttributeType in Master Attributes (attribute_types)
|
|
attr_type = None
|
|
if attr_type_cache is not None and attr_code in attr_type_cache:
|
|
attr_type = attr_type_cache[attr_code]
|
|
else:
|
|
attr_type = db_session.query(AttributeType).filter(AttributeType.code == attr_code).first()
|
|
if not attr_type:
|
|
attr_type = db_session.query(AttributeType).filter(AttributeType.name == attr_name).first()
|
|
|
|
if not attr_type:
|
|
at_id = str(ulid.ULID())
|
|
attr_type = AttributeType(
|
|
attribute_id=at_id,
|
|
name=attr_name,
|
|
code=attr_code,
|
|
status="active"
|
|
)
|
|
db_session.add(attr_type)
|
|
if attr_type_cache is not None:
|
|
attr_type_cache[attr_code] = attr_type
|
|
|
|
# Link VariantAttribute (canonicalizing media_key and parent_media_key values)
|
|
clean_attr_val = str(attr_val)
|
|
if attr_code in ("media_key", "parent_media_key"):
|
|
clean_attr_val = MediaResolver.canonicalize_media_key(attr_val) or clean_attr_val
|
|
|
|
va_id = str(ulid.ULID())
|
|
var_attr = VariantAttribute(
|
|
id=va_id,
|
|
variant_id=variant.variant_id,
|
|
attribute_id=attr_type.attribute_id,
|
|
attribute_value=clean_attr_val
|
|
)
|
|
db_session.add(var_attr)
|
|
|
|
# 6. Attach Variant Images via Canonical Media Key
|
|
raw_media_key = mapped_row.get("media_key") or mapped_row.get("parent_media_key")
|
|
media_key = MediaResolver.canonicalize_media_key(raw_media_key)
|
|
if media_key:
|
|
m_group = None
|
|
if media_group_cache is not None and media_key in media_group_cache:
|
|
m_group = media_group_cache[media_key]
|
|
else:
|
|
m_group = db_session.query(MediaGroup).filter(MediaGroup.media_key == media_key).first()
|
|
if not m_group and mapped_row.get("parent_media_key"):
|
|
p_key = MediaResolver.canonicalize_media_key(mapped_row.get("parent_media_key"))
|
|
if p_key:
|
|
m_group = db_session.query(MediaGroup).filter(MediaGroup.media_key == p_key).first()
|
|
if media_group_cache is not None:
|
|
media_group_cache[media_key] = m_group
|
|
|
|
if m_group and m_group.media_assets:
|
|
db_session.query(VariantImage).filter(VariantImage.variant_id == variant.variant_id).delete()
|
|
for idx, asset in enumerate(m_group.media_assets):
|
|
vi_id = str(ulid.ULID())
|
|
var_img = VariantImage(
|
|
image_id=vi_id,
|
|
variant_id=variant.variant_id,
|
|
image_url=asset.cdn_url,
|
|
sort_order=idx,
|
|
is_primary=(idx == 0)
|
|
)
|
|
db_session.add(var_img)
|
|
# 7. Update inventory ledger stock level
|
|
stock_val = mapped_row.get("stock")
|
|
if stock_val is not None and str(stock_val).strip() != "":
|
|
try:
|
|
stock_qty = int(float(stock_val))
|
|
if stock_qty >= 0:
|
|
from app.services.InventoryService import apply_stock_target
|
|
apply_stock_target(
|
|
variant_id=variant.variant_id,
|
|
target_qty=stock_qty,
|
|
db=db_session,
|
|
notes=f"Imported stock level via migration batch {batch_id}",
|
|
commit=False
|
|
)
|
|
except (ValueError, TypeError):
|
|
pass
|
|
|
|
sp.commit()
|
|
success_count += 1
|
|
|
|
except Exception as e:
|
|
if sp is not None:
|
|
sp.rollback()
|
|
fail_count += 1
|
|
runtime_errors.append({
|
|
"row_number": row_number,
|
|
"sku": sku,
|
|
"severity": "ERROR",
|
|
"field_name": "execution",
|
|
"error_message": f"Database transaction error: {str(e)}",
|
|
"suggested_fix": "Check database constraints and field types"
|
|
})
|
|
|
|
db_session.commit()
|
|
return success_count, fail_count, runtime_errors
|