# pyrefly: ignore [missing-import] import ulid import re from typing import List, Dict, Any, Tuple from decimal import Decimal from sqlalchemy.orm import Session import app.models.db_base from app.models.ProductModel import Product, ProductVariant, VariantImage, AttributeType, VariantAttribute, ProductLocation from app.models.BrandModel import Brand from app.models.CategoryModel import Category from app.models.DeviceCatalogModel import DeviceSeries, DeviceModel, ServiceType, RepairService, RepairVariant from app.models.SystemConfigModel import CrmLocation from app.models.MigrationModel import MediaGroup, MediaAsset, MigrationSnapshot from app.services.migration_engine.media_resolver import MediaResolver from app.utils.html_utils import strip_html BRAND_LOGOS = {} CATEGORY_IMAGES = {} class UpsertExecutor: """ Executes chunked batch transactions according to selected Import Mode: - CREATE_ONLY - UPDATE_EXISTING - UPSERT - SKIP_EXISTING Saves delta snapshots in migration_snapshots for atomic Rollback. Groups multiple variants under single parent Product. Populates Master Attributes (attribute_types) and Variant Attributes (variant_attributes). Attaches variant images via Media Keys. """ @classmethod def execute_batch( cls, batch_id: str, mapped_rows: List[Tuple[int, Dict[str, Any]]], db_session: Session, import_mode: str = "UPSERT", brand_cache: dict = None, category_cache: dict = None, series_cache: dict = None, model_cache: dict = None, product_cache: dict = None, attr_type_cache: dict = None, media_group_cache: dict = None ) -> Tuple[int, int, List[Dict[str, Any]]]: """ Executes database writes for a list of (row_number, mapped_row) items. Returns (success_count, fail_count, list_of_runtime_errors). """ success_count = 0 fail_count = 0 runtime_errors = [] for row_number, mapped_row in mapped_rows: sku = mapped_row.get("sku") if not sku: from app.services.migration_engine.column_mapper import ColumnMapper sku = ColumnMapper.generate_sku(mapped_row) mapped_row["sku"] = sku if not sku: fail_count += 1 runtime_errors.append({ "row_number": row_number, "sku": None, "severity": "ERROR", "field_name": "sku", "error_message": "Missing SKU in execution batch", "suggested_fix": "Ensure row contains a valid SKU" }) continue sp = None try: sp = db_session.begin_nested() # 0. Get device type device_type_val = mapped_row.get("device_type") or (mapped_row.get("attributes") or {}).get("device_type") if device_type_val: device_type_val = str(device_type_val).strip().lower() if device_type_val not in ("laptop", "tablet", "mobile"): device_type_val = None # 1. Resolve Brand & Category brand_name = mapped_row.get("brand") brand_obj = None if brand_name: if brand_cache is not None and brand_name in brand_cache: brand_obj = brand_cache[brand_name] else: brand_obj = db_session.query(Brand).filter(Brand.name == brand_name).first() if not brand_obj: try: b_sp = db_session.begin_nested() b_id = str(ulid.ULID()) brand_obj = Brand( brand_id=b_id, name=brand_name, slug=brand_name.lower().replace(" ", "-"), logo_url=BRAND_LOGOS.get(brand_name), is_active=True ) db_session.add(brand_obj) b_sp.commit() except Exception: try: b_sp.rollback() except Exception: pass brand_obj = db_session.query(Brand).filter(Brand.name == brand_name).first() if brand_cache is not None and brand_obj: brand_cache[brand_name] = brand_obj # Map brand device type dynamically if brand_obj and device_type_val: from app.models.BrandDeviceTypeModel import BrandDeviceType exists_bdt = db_session.query(BrandDeviceType).filter( BrandDeviceType.brand_id == brand_obj.brand_id, BrandDeviceType.device_type == device_type_val ).first() if not exists_bdt: try: bdt_sp = db_session.begin_nested() bdt = BrandDeviceType( brand_id=brand_obj.brand_id, device_type=device_type_val ) db_session.add(bdt) bdt_sp.commit() except Exception: try: bdt_sp.rollback() except Exception: pass cat_name = mapped_row.get("category") or "General" parent_cat_name = mapped_row.get("parent_category") is_parent_feature_val = mapped_row.get("is_parent_feature") is_parent_feature_bool = False if is_parent_feature_val: is_parent_feature_bool = str(is_parent_feature_val).strip().lower() in ("true", "1", "yes") cat_obj = None if category_cache is not None and cat_name in category_cache: cat_obj = category_cache[cat_name] else: cat_obj = db_session.query(Category).filter(Category.name == cat_name).first() if not cat_obj: try: c_sp = db_session.begin_nested() c_id = str(ulid.ULID()) cat_obj = Category( category_id=c_id, name=cat_name, slug=cat_name.lower().replace(" ", "-"), image_url=CATEGORY_IMAGES.get(cat_name), is_parent_feature=is_parent_feature_bool, is_active=True ) db_session.add(cat_obj) c_sp.commit() except Exception: try: c_sp.rollback() except Exception: pass cat_obj = db_session.query(Category).filter(Category.name == cat_name).first() if category_cache is not None and cat_obj: category_cache[cat_name] = cat_obj # Update Category properties if needed if cat_obj: if is_parent_feature_bool and not cat_obj.is_parent_feature: cat_obj.is_parent_feature = True if parent_cat_name: parent_cat_name = str(parent_cat_name).strip() parent_cat = None if category_cache is not None and parent_cat_name in category_cache: parent_cat = category_cache[parent_cat_name] else: parent_cat = db_session.query(Category).filter(Category.name == parent_cat_name).first() if not parent_cat: pc_id = str(ulid.ULID()) parent_cat = Category( category_id=pc_id, name=parent_cat_name, slug=parent_cat_name.lower().replace(" ", "-"), image_url=CATEGORY_IMAGES.get(parent_cat_name), is_active=True ) db_session.add(parent_cat) db_session.flush() if category_cache is not None: category_cache[parent_cat_name] = parent_cat if parent_cat and cat_obj.parent_category_id != parent_cat.category_id: cat_obj.parent_category_id = parent_cat.category_id # 1b. Resolve Global Device Series & Device Model device_model_val = ( mapped_row.get("device_model") or mapped_row.get("model") or (mapped_row.get("attributes") or {}).get("Device Model") or (mapped_row.get("attributes") or {}).get("device_model") or (mapped_row.get("attributes") or {}).get("Model") ) device_series_val = ( mapped_row.get("device_series") or mapped_row.get("series") or (mapped_row.get("attributes") or {}).get("Device Series") or (mapped_row.get("attributes") or {}).get("device_series") or (mapped_row.get("attributes") or {}).get("Series") ) series_obj = None model_obj = None if brand_obj: # Resolve Device Series s_name = device_series_val or f"{brand_obj.name} Series" s_key = f"{brand_obj.brand_id}:{s_name}" if series_cache is not None and s_key in series_cache: series_obj = series_cache[s_key] else: s_slug = s_name.lower().replace(" ", "-").replace("/", "-") series_obj = db_session.query(DeviceSeries).filter( DeviceSeries.brand_id == brand_obj.brand_id, DeviceSeries.name == s_name ).first() if not series_obj: series_obj = db_session.query(DeviceSeries).filter( DeviceSeries.slug == s_slug ).first() if not series_obj: s_id = str(ulid.ULID()) series_obj = DeviceSeries( series_id=s_id, brand_id=brand_obj.brand_id, name=s_name, slug=s_slug, device_type=device_type_val, is_active=True ) db_session.add(series_obj) db_session.flush() if series_cache is not None: series_cache[s_key] = series_obj if series_obj and device_type_val and not series_obj.device_type: series_obj.device_type = device_type_val # Resolve Device Model if device_model_val: m_name = str(device_model_val).strip() m_key = f"{brand_obj.brand_id}:{m_name}" if model_cache is not None and m_key in model_cache: model_obj = model_cache[m_key] else: m_slug = f"{brand_obj.slug}-{m_name.lower().replace(' ', '-').replace('/', '-')}" model_obj = db_session.query(DeviceModel).filter( DeviceModel.brand_id == brand_obj.brand_id, DeviceModel.name == m_name ).first() if not model_obj: model_obj = db_session.query(DeviceModel).filter( DeviceModel.slug == m_slug ).first() if not model_obj: m_id = str(ulid.ULID()) model_obj = DeviceModel( model_id=m_id, series_id=series_obj.series_id, brand_id=brand_obj.brand_id, name=m_name, slug=m_slug, device_type=device_type_val, full_path=f"/{brand_obj.slug}/{series_obj.slug}/{m_slug}", is_active=True ) db_session.add(model_obj) db_session.flush() if model_cache is not None: model_cache[m_key] = model_obj if model_obj and device_type_val and not model_obj.device_type: model_obj.device_type = device_type_val # 2. Query Existing Variant variant = db_session.query(ProductVariant).filter(ProductVariant.sku == sku).first() if variant and import_mode == "SKIP_EXISTING": sp.rollback() continue if not variant and import_mode == "UPDATE_EXISTING": sp.rollback() fail_count += 1 runtime_errors.append({ "row_number": row_number, "sku": sku, "severity": "WARNING", "field_name": "sku", "error_message": f"Skipped SKU '{sku}' - does not exist", "suggested_fix": "Set mode to UPSERT to insert new items" }) continue if variant and import_mode == "CREATE_ONLY": sp.rollback() fail_count += 1 runtime_errors.append({ "row_number": row_number, "sku": sku, "severity": "ERROR", "field_name": "sku", "error_message": f"Cannot create SKU '{sku}' - already exists", "suggested_fix": "Set mode to UPSERT or UPDATE_EXISTING" }) continue # 3. Resolve Parent Product (Group multiple variants under single parent Product) price_val = float(mapped_row.get("price", 0.0) or 0.0) cost_price_val = float(mapped_row.get("cost_price", 0.0) or 0.0) if cost_price_val <= 0: cost_price_val = round(price_val * 0.7, 2) raw_name = mapped_row.get("name") or f"Product {sku}" # Check for explicit parent_name or clean name by stripping trailing variant tags like '(Black - Matte)' parent_title = mapped_row.get("parent_name") if not parent_title: parent_title = re.sub(r'\s*\([^)]*\)$', '', raw_name).strip() parent_slug = parent_title.lower().replace(" ", "-") show_specs_raw = mapped_row.get("show_specifications") show_specs_val = True if show_specs_raw is not None and str(show_specs_raw).strip() != "": show_specs_val = str(show_specs_raw).strip().lower() not in ("false", "0", "no", "off") is_service_raw = mapped_row.get("is_service_booking") is_service_val = False if is_service_raw is not None and str(is_service_raw).strip() != "": is_service_val = str(is_service_raw).strip().lower() in ("true", "1", "yes", "on") service_category_val = mapped_row.get("service_category") # Find or Create Parent Product prod = None if product_cache is not None and parent_slug in product_cache: prod = product_cache[parent_slug] else: prod = db_session.query(Product).filter(Product.slug == parent_slug).first() if not prod: prod = db_session.query(Product).filter(Product.name == parent_title).first() if not prod: desc_val = mapped_row.get("description") raw_seo_title = mapped_row.get("seo_title") raw_seo_desc = mapped_row.get("seo_description") raw_meta_kw = mapped_row.get("meta_keywords") seo_title_val = strip_html(raw_seo_title) or f"Buy {parent_title} | iFixKart" seo_desc_val = strip_html(raw_seo_desc) or (strip_html(desc_val) if desc_val else f"Order {parent_title} online at iFixKart with fast shipping and warranty.") meta_keywords_val = strip_html(raw_meta_kw) or f"{parent_title}, mobile spare parts, replacement, ifixkart" p_id = str(ulid.ULID()) prod = Product( product_id=p_id, name=parent_title, slug=parent_slug, full_path=f"/{cat_obj.slug}/{parent_slug}" if cat_obj else f"/{parent_slug}", brand_id=brand_obj.brand_id if brand_obj else None, device_series_id=series_obj.series_id if series_obj else None, device_model_id=model_obj.model_id if model_obj else None, category_id=cat_obj.category_id if cat_obj else None, device_type=device_type_val, description=str(desc_val) if desc_val else None, warranty_type=str(mapped_row.get("warranty_type")) if mapped_row.get("warranty_type") else None, warranty_summary=str(mapped_row.get("warranty_summary")) if mapped_row.get("warranty_summary") else None, seo_title=str(seo_title_val) if seo_title_val else None, seo_description=str(seo_desc_val) if seo_desc_val else None, meta_keywords=str(meta_keywords_val) if meta_keywords_val else None, show_specifications=show_specs_val, is_service_booking=is_service_val, status="active" ) db_session.add(prod) db_session.flush() else: if series_obj and not prod.device_series_id: prod.device_series_id = series_obj.series_id if model_obj and not prod.device_model_id: prod.device_model_id = model_obj.model_id if device_type_val and not prod.device_type: prod.device_type = device_type_val if mapped_row.get("description"): prod.description = str(mapped_row.get("description")) if mapped_row.get("warranty_type"): prod.warranty_type = str(mapped_row.get("warranty_type")) if mapped_row.get("warranty_summary"): prod.warranty_summary = str(mapped_row.get("warranty_summary")) if mapped_row.get("seo_title"): prod.seo_title = strip_html(mapped_row.get("seo_title")) if mapped_row.get("seo_description"): prod.seo_description = strip_html(mapped_row.get("seo_description")) if mapped_row.get("meta_keywords"): prod.meta_keywords = strip_html(mapped_row.get("meta_keywords")) if mapped_row.get("show_specifications") is not None and str(mapped_row.get("show_specifications")).strip() != "": prod.show_specifications = show_specs_val if mapped_row.get("is_service_booking") is not None and str(mapped_row.get("is_service_booking")).strip() != "": prod.is_service_booking = is_service_val if product_cache is not None: product_cache[parent_slug] = prod # 4. Create or Update Variant if not variant: v_id = str(ulid.ULID()) variant = ProductVariant( variant_id=v_id, product_id=prod.product_id, sku=sku, price=price_val, cost_price=cost_price_val, status="active" ) db_session.add(variant) snapshot = MigrationSnapshot( batch_id=batch_id, entity_table="product_variants", entity_id=variant.variant_id, action_taken="INSERTED", previous_state=None ) db_session.add(snapshot) else: prev_state = { "price": float(variant.price or 0.0), "cost_price": float(variant.cost_price or 0.0), "sku": str(variant.sku) } variant.price = price_val if cost_price_val > 0: variant.cost_price = cost_price_val variant.product_id = prod.product_id snapshot = MigrationSnapshot( batch_id=batch_id, entity_table="product_variants", entity_id=variant.variant_id, action_taken="UPDATED", previous_state=prev_state ) db_session.add(snapshot) # 5. Dynamic Master Attributes (attribute_types) & Variant Attributes (variant_attributes) all_attributes: Dict[str, Any] = {} if "attributes" in mapped_row and isinstance(mapped_row["attributes"], dict): all_attributes.update(mapped_row["attributes"]) if mapped_row.get("media_key"): all_attributes["media_key"] = mapped_row["media_key"] if mapped_row.get("parent_media_key"): all_attributes["parent_media_key"] = mapped_row["parent_media_key"] EXCLUDED_DYNAMIC_KEYS = ( "sku", "name", "parent_name", "price", "cost_price", "stock", "brand", "category", "parent_category", "is_parent_feature", "barcode", "description", "warranty_type", "warranty_summary", "seo_title", "seo_description", "meta_keywords", "attributes", "media_key", "parent_media_key", "device_series", "device_model", "device_type", "show_specifications", "is_service_booking", "service_category", "store_locations", "is_active" ) for key, val in mapped_row.items(): if key not in EXCLUDED_DYNAMIC_KEYS: if val: all_attributes[key] = val if all_attributes: # Clear previous variant_attributes links for this variant db_session.query(VariantAttribute).filter(VariantAttribute.variant_id == variant.variant_id).delete() for attr_name, attr_val in all_attributes.items(): if not attr_val: continue attr_code = attr_name.lower().strip().replace(" ", "_").replace("-", "_") # Find or Create AttributeType in Master Attributes (attribute_types) attr_type = None if attr_type_cache is not None and attr_code in attr_type_cache: attr_type = attr_type_cache[attr_code] else: attr_type = db_session.query(AttributeType).filter(AttributeType.code == attr_code).first() if not attr_type: attr_type = db_session.query(AttributeType).filter(AttributeType.name == attr_name).first() if not attr_type: at_id = str(ulid.ULID()) attr_type = AttributeType( attribute_id=at_id, name=attr_name, code=attr_code, status="active" ) db_session.add(attr_type) if attr_type_cache is not None: attr_type_cache[attr_code] = attr_type # Link VariantAttribute (canonicalizing media_key and parent_media_key values) clean_attr_val = str(attr_val) if attr_code in ("media_key", "parent_media_key"): clean_attr_val = MediaResolver.canonicalize_media_key(attr_val) or clean_attr_val va_id = str(ulid.ULID()) var_attr = VariantAttribute( id=va_id, variant_id=variant.variant_id, attribute_id=attr_type.attribute_id, attribute_value=clean_attr_val ) db_session.add(var_attr) # 6. Attach Variant Images via Smart 3-Tier Canonical Media Key Resolver raw_media_key = mapped_row.get("media_key") or mapped_row.get("parent_media_key") # Tier 2 Fallback: Match by SKU if no media_key column is in Excel if not raw_media_key: raw_media_key = mapped_row.get("sku") # Tier 3 Fallback: Match by Product Name slug if not raw_media_key: raw_media_key = mapped_row.get("name") or mapped_row.get("parent_name") media_key = MediaResolver.canonicalize_media_key(raw_media_key) if media_key: m_group = None if media_group_cache is not None and media_key in media_group_cache: m_group = media_group_cache[media_key] else: m_group = db_session.query(MediaGroup).filter(MediaGroup.media_key == media_key).first() if not m_group and mapped_row.get("parent_media_key"): p_key = MediaResolver.canonicalize_media_key(mapped_row.get("parent_media_key")) if p_key: m_group = db_session.query(MediaGroup).filter(MediaGroup.media_key == p_key).first() if media_group_cache is not None: media_group_cache[media_key] = m_group if m_group and m_group.media_assets: db_session.query(VariantImage).filter(VariantImage.variant_id == variant.variant_id).delete() for idx, asset in enumerate(m_group.media_assets): vi_id = str(ulid.ULID()) var_img = VariantImage( image_id=vi_id, variant_id=variant.variant_id, image_url=asset.cdn_url, sort_order=idx, is_primary=(idx == 0) ) db_session.add(var_img) # 7. Update inventory ledger stock level stock_val = mapped_row.get("stock") if stock_val is not None and str(stock_val).strip() != "": try: stock_qty = int(float(stock_val)) if stock_qty >= 0: from app.services.InventoryService import apply_stock_target apply_stock_target( variant_id=variant.variant_id, target_qty=stock_qty, db=db_session, notes=f"Imported stock level via migration batch {batch_id}", commit=False ) except (ValueError, TypeError): pass # 8. Service Booking Mapping (auto-create RepairService & RepairVariant) if prod.is_service_booking and (service_category_val or prod.mapped_service_category_id): st_obj = None if service_category_val: sc_clean = str(service_category_val).strip() st_obj = db_session.query(ServiceType).filter( (ServiceType.service_type_id == sc_clean) | (ServiceType.name == sc_clean) | (ServiceType.slug == sc_clean.lower().replace(" ", "-")) ).first() if not st_obj: st_obj = db_session.query(ServiceType).filter(ServiceType.name.ilike(f"%{sc_clean}%")).first() if not st_obj: st_id = str(ulid.ULID()) st_slug = sc_clean.lower().replace(" ", "-") st_obj = ServiceType( service_type_id=st_id, name=sc_clean, slug=st_slug, description=f"Auto-created service category for {sc_clean}", is_active=True ) db_session.add(st_obj) db_session.flush() if not st_obj and prod.mapped_service_category_id: st_obj = db_session.query(ServiceType).filter(ServiceType.service_type_id == prod.mapped_service_category_id).first() if st_obj: prod.mapped_service_category_id = st_obj.service_type_id if prod.device_model_id: repair_service = db_session.query(RepairService).filter( RepairService.model_id == prod.device_model_id, RepairService.service_type_id == st_obj.service_type_id ).first() if not repair_service: mod = db_session.query(DeviceModel).filter(DeviceModel.model_id == prod.device_model_id).first() rs_slug = f"{st_obj.slug}-{mod.slug}" if mod else f"{st_obj.slug}-{prod.device_model_id}" rs_full_path = f"/{mod.full_path.strip('/')}/{st_obj.slug}" if mod and mod.full_path else f"/{rs_slug}" repair_service = RepairService( repair_service_id=str(ulid.ULID()), model_id=prod.device_model_id, service_type_id=st_obj.service_type_id, slug=rs_slug, full_path=rs_full_path, description=f"Auto-mapped service from product {prod.name}" ) db_session.add(repair_service) db_session.flush() rv_name = variant.sku if all_attributes: attr_summary = " | ".join([str(v) for k, v in all_attributes.items() if k not in ("media_key", "parent_media_key")]) if attr_summary: rv_name = attr_summary if not rv_name: rv_name = raw_name rv = db_session.query(RepairVariant).filter( RepairVariant.repair_service_id == repair_service.repair_service_id, RepairVariant.name == rv_name[:100] ).first() if not rv: rv = RepairVariant( variant_id=str(ulid.ULID()), repair_service_id=repair_service.repair_service_id, name=rv_name[:100], price=price_val, cost=cost_price_val, duration_minutes=45, warranty_days=90, status="active" ) db_session.add(rv) db_session.flush() else: rv.price = price_val if cost_price_val > 0: rv.cost = cost_price_val # 9. Store Locations & Multi-Branch Inventory Mapping store_locations_raw = mapped_row.get("store_locations") if store_locations_raw and str(store_locations_raw).strip() != "": loc_tokens = [tok.strip() for tok in re.split(r'[;,]+', str(store_locations_raw)) if tok.strip()] for token in loc_tokens: parts = [p.strip() for p in token.split(":")] loc_code = parts[0] loc_qty = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else (int(float(stock_val)) if stock_val is not None and str(stock_val).strip() != "" else 0) loc_selling = Decimal(str(parts[2])) if len(parts) > 2 else Decimal(str(price_val)) loc_cost = Decimal(str(cost_price_val)) crm_loc = db_session.query(CrmLocation).filter( (CrmLocation.code == loc_code) | (CrmLocation.name == loc_code) | (CrmLocation.location_id == loc_code) ).first() if crm_loc: prod_loc = db_session.query(ProductLocation).filter( ProductLocation.product_id == prod.product_id, ProductLocation.location_id == crm_loc.location_id ).first() if not prod_loc: prod_loc = ProductLocation( mapping_id=str(ulid.ULID()), tenant_id=crm_loc.tenant_id or "tenant_default", product_id=prod.product_id, location_id=crm_loc.location_id, stock_quantity=loc_qty, reserved_quantity=0, reorder_level=5, cost_price=loc_cost, selling_price=loc_selling, is_active=True ) db_session.add(prod_loc) else: prod_loc.stock_quantity = loc_qty prod_loc.selling_price = loc_selling prod_loc.cost_price = loc_cost prod_loc.is_active = True sp.commit() success_count += 1 except Exception as e: if sp is not None: try: sp.rollback() except Exception: pass fail_count += 1 runtime_errors.append({ "row_number": row_number, "sku": sku, "severity": "ERROR", "field_name": "execution", "error_message": f"Database transaction error: {str(e)}", "suggested_fix": "Check database constraints and field types" }) db_session.commit() return success_count, fail_count, runtime_errors