# pyrefly: ignore [missing-import] import ulid import re from typing import List, Dict, Any, Tuple from sqlalchemy.orm import Session import app.models.db_base from app.models.ProductModel import Product, ProductVariant, VariantImage, AttributeType, VariantAttribute from app.models.BrandModel import Brand from app.models.CategoryModel import Category from app.models.DeviceCatalogModel import DeviceSeries, DeviceModel from app.models.MigrationModel import MediaGroup, MediaAsset, MigrationSnapshot from app.services.migration_engine.media_resolver import MediaResolver BRAND_LOGOS = {} CATEGORY_IMAGES = {} class UpsertExecutor: """ Executes chunked batch transactions according to selected Import Mode: - CREATE_ONLY - UPDATE_EXISTING - UPSERT - SKIP_EXISTING Saves delta snapshots in migration_snapshots for atomic Rollback. Groups multiple variants under single parent Product. Populates Master Attributes (attribute_types) and Variant Attributes (variant_attributes). Attaches variant images via Media Keys. """ @classmethod def execute_batch( cls, batch_id: str, mapped_rows: List[Tuple[int, Dict[str, Any]]], db_session: Session, import_mode: str = "UPSERT", brand_cache: dict = None, category_cache: dict = None, series_cache: dict = None, model_cache: dict = None, product_cache: dict = None, attr_type_cache: dict = None, media_group_cache: dict = None ) -> Tuple[int, int, List[Dict[str, Any]]]: """ Executes database writes for a list of (row_number, mapped_row) items. Returns (success_count, fail_count, list_of_runtime_errors). """ success_count = 0 fail_count = 0 runtime_errors = [] for row_number, mapped_row in mapped_rows: sku = mapped_row.get("sku") if not sku: fail_count += 1 runtime_errors.append({ "row_number": row_number, "sku": None, "severity": "ERROR", "field_name": "sku", "error_message": "Missing SKU in execution batch", "suggested_fix": "Ensure row contains a valid SKU" }) continue sp = None try: sp = db_session.begin_nested() # 0. Get device type device_type_val = mapped_row.get("device_type") or (mapped_row.get("attributes") or {}).get("device_type") if device_type_val: device_type_val = str(device_type_val).strip().lower() if device_type_val not in ("laptop", "tablet", "mobile"): device_type_val = None # 1. Resolve Brand & Category brand_name = mapped_row.get("brand") brand_obj = None if brand_name: if brand_cache is not None and brand_name in brand_cache: brand_obj = brand_cache[brand_name] else: brand_obj = db_session.query(Brand).filter(Brand.name == brand_name).first() if not brand_obj: b_id = str(ulid.ULID()) brand_obj = Brand( brand_id=b_id, name=brand_name, slug=brand_name.lower().replace(" ", "-"), logo_url=BRAND_LOGOS.get(brand_name), is_active=True ) db_session.add(brand_obj) db_session.flush() if brand_cache is not None: brand_cache[brand_name] = brand_obj # Map brand device type dynamically if brand_obj and device_type_val: from app.models.BrandDeviceTypeModel import BrandDeviceType exists_bdt = db_session.query(BrandDeviceType).filter( BrandDeviceType.brand_id == brand_obj.brand_id, BrandDeviceType.device_type == device_type_val ).first() if not exists_bdt: bdt = BrandDeviceType( brand_id=brand_obj.brand_id, device_type=device_type_val ) db_session.add(bdt) cat_name = mapped_row.get("category") or "General" parent_cat_name = mapped_row.get("parent_category") is_parent_feature_val = mapped_row.get("is_parent_feature") is_parent_feature_bool = False if is_parent_feature_val: is_parent_feature_bool = str(is_parent_feature_val).strip().lower() in ("true", "1", "yes") cat_obj = None if category_cache is not None and cat_name in category_cache: cat_obj = category_cache[cat_name] else: cat_obj = db_session.query(Category).filter(Category.name == cat_name).first() if not cat_obj: c_id = str(ulid.ULID()) cat_obj = Category( category_id=c_id, name=cat_name, slug=cat_name.lower().replace(" ", "-"), image_url=CATEGORY_IMAGES.get(cat_name), is_parent_feature=is_parent_feature_bool, is_active=True ) db_session.add(cat_obj) db_session.flush() if category_cache is not None: category_cache[cat_name] = cat_obj # Update Category properties if needed if cat_obj: if is_parent_feature_bool and not cat_obj.is_parent_feature: cat_obj.is_parent_feature = True if parent_cat_name: parent_cat_name = str(parent_cat_name).strip() parent_cat = None if category_cache is not None and parent_cat_name in category_cache: parent_cat = category_cache[parent_cat_name] else: parent_cat = db_session.query(Category).filter(Category.name == parent_cat_name).first() if not parent_cat: pc_id = str(ulid.ULID()) parent_cat = Category( category_id=pc_id, name=parent_cat_name, slug=parent_cat_name.lower().replace(" ", "-"), image_url=CATEGORY_IMAGES.get(parent_cat_name), is_active=True ) db_session.add(parent_cat) db_session.flush() if category_cache is not None: category_cache[parent_cat_name] = parent_cat if parent_cat and cat_obj.parent_category_id != parent_cat.category_id: cat_obj.parent_category_id = parent_cat.category_id # 1b. Resolve Global Device Series & Device Model device_model_val = ( mapped_row.get("device_model") or mapped_row.get("model") or (mapped_row.get("attributes") or {}).get("Device Model") or (mapped_row.get("attributes") or {}).get("device_model") or (mapped_row.get("attributes") or {}).get("Model") ) device_series_val = ( mapped_row.get("device_series") or mapped_row.get("series") or (mapped_row.get("attributes") or {}).get("Device Series") or (mapped_row.get("attributes") or {}).get("device_series") or (mapped_row.get("attributes") or {}).get("Series") ) series_obj = None model_obj = None if brand_obj: # Resolve Device Series s_name = device_series_val or f"{brand_obj.name} Series" s_key = f"{brand_obj.brand_id}:{s_name}" if series_cache is not None and s_key in series_cache: series_obj = series_cache[s_key] else: s_slug = s_name.lower().replace(" ", "-").replace("/", "-") series_obj = db_session.query(DeviceSeries).filter( DeviceSeries.brand_id == brand_obj.brand_id, DeviceSeries.name == s_name ).first() if not series_obj: series_obj = db_session.query(DeviceSeries).filter( DeviceSeries.slug == s_slug ).first() if not series_obj: s_id = str(ulid.ULID()) series_obj = DeviceSeries( series_id=s_id, brand_id=brand_obj.brand_id, name=s_name, slug=s_slug, device_type=device_type_val, is_active=True ) db_session.add(series_obj) db_session.flush() if series_cache is not None: series_cache[s_key] = series_obj if series_obj and device_type_val and not series_obj.device_type: series_obj.device_type = device_type_val # Resolve Device Model if device_model_val: m_name = str(device_model_val).strip() m_key = f"{brand_obj.brand_id}:{m_name}" if model_cache is not None and m_key in model_cache: model_obj = model_cache[m_key] else: m_slug = f"{brand_obj.slug}-{m_name.lower().replace(' ', '-').replace('/', '-')}" model_obj = db_session.query(DeviceModel).filter( DeviceModel.brand_id == brand_obj.brand_id, DeviceModel.name == m_name ).first() if not model_obj: model_obj = db_session.query(DeviceModel).filter( DeviceModel.slug == m_slug ).first() if not model_obj: m_id = str(ulid.ULID()) model_obj = DeviceModel( model_id=m_id, series_id=series_obj.series_id, brand_id=brand_obj.brand_id, name=m_name, slug=m_slug, device_type=device_type_val, full_path=f"/{brand_obj.slug}/{series_obj.slug}/{m_slug}", is_active=True ) db_session.add(model_obj) db_session.flush() if model_cache is not None: model_cache[m_key] = model_obj if model_obj and device_type_val and not model_obj.device_type: model_obj.device_type = device_type_val # 2. Query Existing Variant variant = db_session.query(ProductVariant).filter(ProductVariant.sku == sku).first() if variant and import_mode == "SKIP_EXISTING": sp.rollback() continue if not variant and import_mode == "UPDATE_EXISTING": sp.rollback() fail_count += 1 runtime_errors.append({ "row_number": row_number, "sku": sku, "severity": "WARNING", "field_name": "sku", "error_message": f"Skipped SKU '{sku}' - does not exist", "suggested_fix": "Set mode to UPSERT to insert new items" }) continue if variant and import_mode == "CREATE_ONLY": sp.rollback() fail_count += 1 runtime_errors.append({ "row_number": row_number, "sku": sku, "severity": "ERROR", "field_name": "sku", "error_message": f"Cannot create SKU '{sku}' - already exists", "suggested_fix": "Set mode to UPSERT or UPDATE_EXISTING" }) continue # 3. Resolve Parent Product (Group multiple variants under single parent Product) price_val = float(mapped_row.get("price", 0.0) or 0.0) raw_name = mapped_row.get("name") or f"Product {sku}" # Check for explicit parent_name or clean name by stripping trailing variant tags like '(Black - Matte)' parent_title = mapped_row.get("parent_name") if not parent_title: parent_title = re.sub(r'\s*\([^)]*\)$', '', raw_name).strip() parent_slug = parent_title.lower().replace(" ", "-") # Find or Create Parent Product prod = None if product_cache is not None and parent_slug in product_cache: prod = product_cache[parent_slug] else: prod = db_session.query(Product).filter(Product.slug == parent_slug).first() if not prod: prod = db_session.query(Product).filter(Product.name == parent_title).first() if not prod: p_id = str(ulid.ULID()) prod = Product( product_id=p_id, name=parent_title, slug=parent_slug, full_path=f"/{cat_obj.slug}/{parent_slug}", brand_id=brand_obj.brand_id if brand_obj else None, device_series_id=series_obj.series_id if series_obj else None, device_model_id=model_obj.model_id if model_obj else None, category_id=cat_obj.category_id, device_type=device_type_val, status="active" ) db_session.add(prod) db_session.flush() else: if series_obj and not prod.device_series_id: prod.device_series_id = series_obj.series_id if model_obj and not prod.device_model_id: prod.device_model_id = model_obj.model_id if device_type_val and not prod.device_type: prod.device_type = device_type_val if product_cache is not None: product_cache[parent_slug] = prod # 4. Create or Update Variant if not variant: v_id = str(ulid.ULID()) variant = ProductVariant( variant_id=v_id, product_id=prod.product_id, sku=sku, price=price_val, cost_price=price_val * 0.7, status="active" ) db_session.add(variant) snapshot = MigrationSnapshot( batch_id=batch_id, entity_table="product_variants", entity_id=variant.variant_id, action_taken="INSERTED", previous_state=None ) db_session.add(snapshot) else: prev_state = { "price": float(variant.price or 0.0), "sku": str(variant.sku) } variant.price = price_val variant.product_id = prod.product_id snapshot = MigrationSnapshot( batch_id=batch_id, entity_table="product_variants", entity_id=variant.variant_id, action_taken="UPDATED", previous_state=prev_state ) db_session.add(snapshot) # 5. Dynamic Master Attributes (attribute_types) & Variant Attributes (variant_attributes) # Collect attributes from mapped_row.get("attributes") or key-values in row all_attributes: Dict[str, Any] = {} if "attributes" in mapped_row and isinstance(mapped_row["attributes"], dict): all_attributes.update(mapped_row["attributes"]) for key, val in mapped_row.items(): if key not in ("sku", "name", "parent_name", "price", "stock", "brand", "category", "barcode", "description", "attributes"): if val: all_attributes[key] = val if all_attributes: # Clear previous variant_attributes links for this variant db_session.query(VariantAttribute).filter(VariantAttribute.variant_id == variant.variant_id).delete() for attr_name, attr_val in all_attributes.items(): if not attr_val: continue attr_code = attr_name.lower().strip().replace(" ", "_").replace("-", "_") # Find or Create AttributeType in Master Attributes (attribute_types) attr_type = None if attr_type_cache is not None and attr_code in attr_type_cache: attr_type = attr_type_cache[attr_code] else: attr_type = db_session.query(AttributeType).filter(AttributeType.code == attr_code).first() if not attr_type: attr_type = db_session.query(AttributeType).filter(AttributeType.name == attr_name).first() if not attr_type: at_id = str(ulid.ULID()) attr_type = AttributeType( attribute_id=at_id, name=attr_name, code=attr_code, status="active" ) db_session.add(attr_type) if attr_type_cache is not None: attr_type_cache[attr_code] = attr_type # Link VariantAttribute (canonicalizing media_key and parent_media_key values) clean_attr_val = str(attr_val) if attr_code in ("media_key", "parent_media_key"): clean_attr_val = MediaResolver.canonicalize_media_key(attr_val) or clean_attr_val va_id = str(ulid.ULID()) var_attr = VariantAttribute( id=va_id, variant_id=variant.variant_id, attribute_id=attr_type.attribute_id, attribute_value=clean_attr_val ) db_session.add(var_attr) # 6. Attach Variant Images via Canonical Media Key raw_media_key = mapped_row.get("media_key") or mapped_row.get("parent_media_key") media_key = MediaResolver.canonicalize_media_key(raw_media_key) if media_key: m_group = None if media_group_cache is not None and media_key in media_group_cache: m_group = media_group_cache[media_key] else: m_group = db_session.query(MediaGroup).filter(MediaGroup.media_key == media_key).first() if not m_group and mapped_row.get("parent_media_key"): p_key = MediaResolver.canonicalize_media_key(mapped_row.get("parent_media_key")) if p_key: m_group = db_session.query(MediaGroup).filter(MediaGroup.media_key == p_key).first() if media_group_cache is not None: media_group_cache[media_key] = m_group if m_group and m_group.media_assets: db_session.query(VariantImage).filter(VariantImage.variant_id == variant.variant_id).delete() for idx, asset in enumerate(m_group.media_assets): vi_id = str(ulid.ULID()) var_img = VariantImage( image_id=vi_id, variant_id=variant.variant_id, image_url=asset.cdn_url, sort_order=idx, is_primary=(idx == 0) ) db_session.add(var_img) # 7. Update inventory ledger stock level stock_val = mapped_row.get("stock") if stock_val is not None and str(stock_val).strip() != "": try: stock_qty = int(float(stock_val)) if stock_qty >= 0: from app.services.InventoryService import apply_stock_target apply_stock_target( variant_id=variant.variant_id, target_qty=stock_qty, db=db_session, notes=f"Imported stock level via migration batch {batch_id}", commit=False ) except (ValueError, TypeError): pass sp.commit() success_count += 1 except Exception as e: if sp is not None: sp.rollback() fail_count += 1 runtime_errors.append({ "row_number": row_number, "sku": sku, "severity": "ERROR", "field_name": "execution", "error_message": f"Database transaction error: {str(e)}", "suggested_fix": "Check database constraints and field types" }) db_session.commit() return success_count, fail_count, runtime_errors