821 lines
43 KiB
Python
821 lines
43 KiB
Python
|
|
# pyrefly: ignore [missing-import]
|
|
import ulid
|
|
import re
|
|
from typing import List, Dict, Any, Tuple
|
|
from decimal import Decimal
|
|
from sqlalchemy.orm import Session
|
|
import app.models.db_base
|
|
from app.models.ProductModel import Product, ProductVariant, VariantImage, AttributeType, VariantAttribute, ProductLocation
|
|
from app.models.BrandModel import Brand
|
|
from app.models.CategoryModel import Category
|
|
from app.models.DeviceCatalogModel import DeviceSeries, DeviceModel, ServiceType, RepairService, RepairVariant
|
|
from app.models.SystemConfigModel import CrmLocation
|
|
from app.models.MigrationModel import MediaGroup, MediaAsset, MigrationSnapshot
|
|
from app.services.migration_engine.media_resolver import MediaResolver
|
|
from app.utils.html_utils import strip_html
|
|
|
|
BRAND_LOGOS = {}
|
|
|
|
CATEGORY_IMAGES = {}
|
|
|
|
|
|
def _safe_slug(text: str) -> str:
|
|
if not text:
|
|
return ""
|
|
clean = str(text).lower().replace("&", "and").replace("/", "-")
|
|
clean = re.sub(r'[^a-z0-9\-]+', '-', clean)
|
|
clean = re.sub(r'-+', '-', clean).strip('-')
|
|
return clean
|
|
|
|
|
|
class UpsertExecutor:
|
|
"""
|
|
Executes chunked batch transactions according to selected Import Mode:
|
|
- CREATE_ONLY
|
|
- UPDATE_EXISTING
|
|
- UPSERT
|
|
- SKIP_EXISTING
|
|
|
|
Saves delta snapshots in migration_snapshots for atomic Rollback.
|
|
Groups multiple variants under single parent Product.
|
|
Populates Master Attributes (attribute_types) and Variant Attributes (variant_attributes).
|
|
Attaches variant images via Media Keys.
|
|
"""
|
|
|
|
@classmethod
|
|
def execute_batch(
|
|
cls,
|
|
batch_id: str,
|
|
mapped_rows: List[Tuple[int, Dict[str, Any]]],
|
|
db_session: Session,
|
|
import_mode: str = "UPSERT",
|
|
brand_cache: dict = None,
|
|
category_cache: dict = None,
|
|
series_cache: dict = None,
|
|
model_cache: dict = None,
|
|
product_cache: dict = None,
|
|
attr_type_cache: dict = None,
|
|
media_group_cache: dict = None
|
|
) -> Tuple[int, int, List[Dict[str, Any]]]:
|
|
"""
|
|
Executes database writes for a list of (row_number, mapped_row) items.
|
|
Returns (success_count, fail_count, list_of_runtime_errors).
|
|
"""
|
|
success_count = 0
|
|
fail_count = 0
|
|
runtime_errors = []
|
|
|
|
for row_number, mapped_row in mapped_rows:
|
|
sku = mapped_row.get("sku")
|
|
if not sku:
|
|
from app.services.migration_engine.column_mapper import ColumnMapper
|
|
sku = ColumnMapper.generate_sku(mapped_row)
|
|
mapped_row["sku"] = sku
|
|
if not sku:
|
|
fail_count += 1
|
|
runtime_errors.append({
|
|
"row_number": row_number,
|
|
"sku": None,
|
|
"severity": "ERROR",
|
|
"field_name": "sku",
|
|
"error_message": "Missing SKU in execution batch",
|
|
"suggested_fix": "Ensure row contains a valid SKU"
|
|
})
|
|
continue
|
|
|
|
sp = None
|
|
new_series_keys = []
|
|
new_model_keys = []
|
|
try:
|
|
sp = db_session.begin_nested()
|
|
# 0. Get device type
|
|
device_type_val = mapped_row.get("device_type") or (mapped_row.get("attributes") or {}).get("device_type")
|
|
if device_type_val:
|
|
device_type_val = str(device_type_val).strip().lower()
|
|
if device_type_val not in ("laptop", "tablet", "mobile"):
|
|
device_type_val = None
|
|
|
|
# 1. Resolve Brand & Category
|
|
brand_name = mapped_row.get("brand")
|
|
brand_obj = None
|
|
if brand_name:
|
|
if brand_cache is not None and brand_name in brand_cache:
|
|
brand_obj = brand_cache[brand_name]
|
|
else:
|
|
brand_obj = db_session.query(Brand).filter(Brand.name == brand_name).first()
|
|
if not brand_obj:
|
|
try:
|
|
b_sp = db_session.begin_nested()
|
|
b_id = str(ulid.ULID())
|
|
brand_obj = Brand(
|
|
brand_id=b_id,
|
|
name=brand_name,
|
|
slug=_safe_slug(brand_name),
|
|
logo_url=BRAND_LOGOS.get(brand_name),
|
|
is_active=True
|
|
)
|
|
db_session.add(brand_obj)
|
|
b_sp.commit()
|
|
except Exception:
|
|
try:
|
|
b_sp.rollback()
|
|
except Exception:
|
|
pass
|
|
brand_obj = db_session.query(Brand).filter(Brand.name == brand_name).first()
|
|
if brand_cache is not None and brand_obj:
|
|
brand_cache[brand_name] = brand_obj
|
|
|
|
# Map brand device type dynamically
|
|
if brand_obj and device_type_val:
|
|
from app.models.BrandDeviceTypeModel import BrandDeviceType
|
|
exists_bdt = db_session.query(BrandDeviceType).filter(
|
|
BrandDeviceType.brand_id == brand_obj.brand_id,
|
|
BrandDeviceType.device_type == device_type_val
|
|
).first()
|
|
if not exists_bdt:
|
|
try:
|
|
bdt_sp = db_session.begin_nested()
|
|
bdt = BrandDeviceType(
|
|
brand_id=brand_obj.brand_id,
|
|
device_type=device_type_val
|
|
)
|
|
db_session.add(bdt)
|
|
bdt_sp.commit()
|
|
except Exception:
|
|
try:
|
|
bdt_sp.rollback()
|
|
except Exception:
|
|
pass
|
|
|
|
cat_name = mapped_row.get("category") or "General"
|
|
parent_cat_name = mapped_row.get("parent_category")
|
|
is_parent_feature_val = mapped_row.get("is_parent_feature")
|
|
is_parent_feature_bool = False
|
|
if is_parent_feature_val:
|
|
is_parent_feature_bool = str(is_parent_feature_val).strip().lower() in ("true", "1", "yes")
|
|
|
|
cat_obj = None
|
|
if category_cache is not None and cat_name in category_cache:
|
|
cat_obj = category_cache[cat_name]
|
|
else:
|
|
cat_obj = db_session.query(Category).filter(Category.name == cat_name).first()
|
|
if not cat_obj:
|
|
try:
|
|
c_sp = db_session.begin_nested()
|
|
c_id = str(ulid.ULID())
|
|
cat_obj = Category(
|
|
category_id=c_id,
|
|
name=cat_name,
|
|
slug=_safe_slug(cat_name),
|
|
image_url=CATEGORY_IMAGES.get(cat_name),
|
|
is_parent_feature=is_parent_feature_bool,
|
|
is_active=True
|
|
)
|
|
db_session.add(cat_obj)
|
|
c_sp.commit()
|
|
except Exception:
|
|
try:
|
|
c_sp.rollback()
|
|
except Exception:
|
|
pass
|
|
cat_obj = db_session.query(Category).filter(Category.name == cat_name).first()
|
|
if category_cache is not None and cat_obj:
|
|
category_cache[cat_name] = cat_obj
|
|
|
|
# Update Category properties if needed
|
|
if cat_obj:
|
|
if is_parent_feature_bool and not cat_obj.is_parent_feature:
|
|
cat_obj.is_parent_feature = True
|
|
|
|
if parent_cat_name:
|
|
parent_cat_name = str(parent_cat_name).strip()
|
|
parent_cat = None
|
|
if category_cache is not None and parent_cat_name in category_cache:
|
|
parent_cat = category_cache[parent_cat_name]
|
|
else:
|
|
parent_cat = db_session.query(Category).filter(Category.name == parent_cat_name).first()
|
|
if not parent_cat:
|
|
pc_id = str(ulid.ULID())
|
|
parent_cat = Category(
|
|
category_id=pc_id,
|
|
name=parent_cat_name,
|
|
slug=_safe_slug(parent_cat_name),
|
|
image_url=CATEGORY_IMAGES.get(parent_cat_name),
|
|
is_active=True
|
|
)
|
|
db_session.add(parent_cat)
|
|
db_session.flush()
|
|
if category_cache is not None:
|
|
category_cache[parent_cat_name] = parent_cat
|
|
|
|
if parent_cat and cat_obj.parent_category_id != parent_cat.category_id:
|
|
cat_obj.parent_category_id = parent_cat.category_id
|
|
|
|
# 1b. Resolve Global Device Series & Device Model
|
|
device_model_val = (
|
|
mapped_row.get("device_model")
|
|
or mapped_row.get("model")
|
|
or (mapped_row.get("attributes") or {}).get("Device Model")
|
|
or (mapped_row.get("attributes") or {}).get("device_model")
|
|
or (mapped_row.get("attributes") or {}).get("Model")
|
|
)
|
|
device_series_val = (
|
|
mapped_row.get("device_series")
|
|
or mapped_row.get("series")
|
|
or (mapped_row.get("attributes") or {}).get("Device Series")
|
|
or (mapped_row.get("attributes") or {}).get("device_series")
|
|
or (mapped_row.get("attributes") or {}).get("Series")
|
|
)
|
|
|
|
series_obj = None
|
|
model_obj = None
|
|
|
|
if brand_obj:
|
|
# Resolve Device Series
|
|
s_name = device_series_val or f"{brand_obj.name} Series"
|
|
s_key = f"{brand_obj.brand_id}:{s_name}"
|
|
if series_cache is not None and s_key in series_cache:
|
|
series_obj = series_cache[s_key]
|
|
else:
|
|
s_slug = s_name.lower().replace(" ", "-").replace("/", "-")
|
|
series_obj = db_session.query(DeviceSeries).filter(
|
|
DeviceSeries.brand_id == brand_obj.brand_id,
|
|
DeviceSeries.name == s_name
|
|
).first()
|
|
if not series_obj:
|
|
series_obj = db_session.query(DeviceSeries).filter(
|
|
DeviceSeries.slug == s_slug
|
|
).first()
|
|
if not series_obj:
|
|
s_id = str(ulid.ULID())
|
|
series_obj = DeviceSeries(
|
|
series_id=s_id,
|
|
brand_id=brand_obj.brand_id,
|
|
name=s_name,
|
|
slug=s_slug,
|
|
device_type=device_type_val,
|
|
is_active=True
|
|
)
|
|
db_session.add(series_obj)
|
|
db_session.flush()
|
|
if series_cache is not None:
|
|
if s_key not in series_cache:
|
|
new_series_keys.append(s_key)
|
|
series_cache[s_key] = series_obj
|
|
|
|
if series_obj and device_type_val and not series_obj.device_type:
|
|
series_obj.device_type = device_type_val
|
|
|
|
# Resolve Device Model
|
|
if device_model_val:
|
|
m_name = str(device_model_val).strip()
|
|
m_key = f"{brand_obj.brand_id}:{m_name}"
|
|
if model_cache is not None and m_key in model_cache:
|
|
model_obj = model_cache[m_key]
|
|
else:
|
|
m_slug = f"{brand_obj.slug}-{m_name.lower().replace(' ', '-').replace('/', '-')}"
|
|
model_obj = db_session.query(DeviceModel).filter(
|
|
DeviceModel.brand_id == brand_obj.brand_id,
|
|
DeviceModel.name == m_name
|
|
).first()
|
|
if not model_obj:
|
|
model_obj = db_session.query(DeviceModel).filter(
|
|
DeviceModel.slug == m_slug
|
|
).first()
|
|
if not model_obj:
|
|
m_id = str(ulid.ULID())
|
|
model_obj = DeviceModel(
|
|
model_id=m_id,
|
|
series_id=series_obj.series_id,
|
|
brand_id=brand_obj.brand_id,
|
|
name=m_name,
|
|
slug=m_slug,
|
|
device_type=device_type_val,
|
|
full_path=f"/{brand_obj.slug}/{series_obj.slug}/{m_slug}",
|
|
is_active=True
|
|
)
|
|
db_session.add(model_obj)
|
|
db_session.flush()
|
|
if model_cache is not None:
|
|
if m_key not in model_cache:
|
|
new_model_keys.append(m_key)
|
|
model_cache[m_key] = model_obj
|
|
|
|
if model_obj and device_type_val and not model_obj.device_type:
|
|
model_obj.device_type = device_type_val
|
|
|
|
# 2. Query Existing Variant
|
|
variant = db_session.query(ProductVariant).filter(ProductVariant.sku == sku).first()
|
|
|
|
if variant and import_mode == "SKIP_EXISTING":
|
|
sp.rollback()
|
|
continue
|
|
|
|
if not variant and import_mode == "UPDATE_EXISTING":
|
|
sp.rollback()
|
|
fail_count += 1
|
|
runtime_errors.append({
|
|
"row_number": row_number,
|
|
"sku": sku,
|
|
"severity": "WARNING",
|
|
"field_name": "sku",
|
|
"error_message": f"Skipped SKU '{sku}' - does not exist",
|
|
"suggested_fix": "Set mode to UPSERT to insert new items"
|
|
})
|
|
continue
|
|
|
|
if variant and import_mode == "CREATE_ONLY":
|
|
sp.rollback()
|
|
fail_count += 1
|
|
runtime_errors.append({
|
|
"row_number": row_number,
|
|
"sku": sku,
|
|
"severity": "ERROR",
|
|
"field_name": "sku",
|
|
"error_message": f"Cannot create SKU '{sku}' - already exists",
|
|
"suggested_fix": "Set mode to UPSERT or UPDATE_EXISTING"
|
|
})
|
|
continue
|
|
|
|
# 3. Resolve Parent Product (Group multiple variants under single parent Product)
|
|
price_val = float(mapped_row.get("price", 0.0) or 0.0)
|
|
cost_price_val = float(mapped_row.get("cost_price", 0.0) or 0.0)
|
|
if cost_price_val <= 0:
|
|
cost_price_val = round(price_val * 0.7, 2)
|
|
|
|
raw_name = mapped_row.get("name") or f"Product {sku}"
|
|
|
|
# Check for explicit parent_name or clean name by stripping trailing variant tags like '(Black - Matte)'
|
|
parent_title = mapped_row.get("parent_name")
|
|
if not parent_title:
|
|
parent_title = re.sub(r'\s*\([^)]*\)$', '', raw_name).strip()
|
|
|
|
parent_slug = parent_title.lower().replace(" ", "-")
|
|
|
|
show_specs_raw = mapped_row.get("show_specifications")
|
|
show_specs_val = True
|
|
if show_specs_raw is not None and str(show_specs_raw).strip() != "":
|
|
show_specs_val = str(show_specs_raw).strip().lower() not in ("false", "0", "no", "off")
|
|
|
|
is_service_raw = mapped_row.get("is_service_booking")
|
|
is_service_val = False
|
|
if is_service_raw is not None and str(is_service_raw).strip() != "":
|
|
is_service_val = str(is_service_raw).strip().lower() in ("true", "1", "yes", "on")
|
|
|
|
service_category_val = mapped_row.get("service_category")
|
|
|
|
# Find or Create Parent Product
|
|
prod = None
|
|
if product_cache is not None and parent_slug in product_cache:
|
|
prod = product_cache[parent_slug]
|
|
else:
|
|
prod = db_session.query(Product).filter(Product.slug == parent_slug).first()
|
|
if not prod:
|
|
prod = db_session.query(Product).filter(Product.name == parent_title).first()
|
|
|
|
if not prod:
|
|
desc_val = mapped_row.get("description")
|
|
raw_seo_title = mapped_row.get("seo_title")
|
|
raw_seo_desc = mapped_row.get("seo_description")
|
|
raw_meta_kw = mapped_row.get("meta_keywords")
|
|
|
|
seo_title_val = strip_html(raw_seo_title) or f"Buy {parent_title} | iFixKart"
|
|
seo_desc_val = strip_html(raw_seo_desc) or (strip_html(desc_val) if desc_val else f"Order {parent_title} online at iFixKart with fast shipping and warranty.")
|
|
meta_keywords_val = strip_html(raw_meta_kw) or f"{parent_title}, mobile spare parts, replacement, ifixkart"
|
|
|
|
p_id = str(ulid.ULID())
|
|
prod = Product(
|
|
product_id=p_id,
|
|
name=parent_title,
|
|
slug=parent_slug,
|
|
full_path=f"/{cat_obj.slug}/{parent_slug}" if cat_obj else f"/{parent_slug}",
|
|
brand_id=brand_obj.brand_id if brand_obj else None,
|
|
device_series_id=series_obj.series_id if series_obj else None,
|
|
device_model_id=model_obj.model_id if model_obj else None,
|
|
category_id=cat_obj.category_id if cat_obj else None,
|
|
device_type=device_type_val,
|
|
description=str(desc_val) if desc_val else None,
|
|
warranty_type=str(mapped_row.get("warranty_type")) if mapped_row.get("warranty_type") else None,
|
|
warranty_summary=str(mapped_row.get("warranty_summary")) if mapped_row.get("warranty_summary") else None,
|
|
seo_title=str(seo_title_val) if seo_title_val else None,
|
|
seo_description=str(seo_desc_val) if seo_desc_val else None,
|
|
meta_keywords=str(meta_keywords_val) if meta_keywords_val else None,
|
|
show_specifications=show_specs_val,
|
|
is_service_booking=is_service_val,
|
|
status="active"
|
|
)
|
|
db_session.add(prod)
|
|
db_session.flush()
|
|
else:
|
|
if series_obj and not prod.device_series_id:
|
|
prod.device_series_id = series_obj.series_id
|
|
if model_obj and not prod.device_model_id:
|
|
prod.device_model_id = model_obj.model_id
|
|
if device_type_val and not prod.device_type:
|
|
prod.device_type = device_type_val
|
|
if mapped_row.get("description"):
|
|
prod.description = str(mapped_row.get("description"))
|
|
if mapped_row.get("warranty_type"):
|
|
prod.warranty_type = str(mapped_row.get("warranty_type"))
|
|
if mapped_row.get("warranty_summary"):
|
|
prod.warranty_summary = str(mapped_row.get("warranty_summary"))
|
|
if mapped_row.get("seo_title"):
|
|
prod.seo_title = strip_html(mapped_row.get("seo_title"))
|
|
if mapped_row.get("seo_description"):
|
|
prod.seo_description = strip_html(mapped_row.get("seo_description"))
|
|
if mapped_row.get("meta_keywords"):
|
|
prod.meta_keywords = strip_html(mapped_row.get("meta_keywords"))
|
|
if mapped_row.get("show_specifications") is not None and str(mapped_row.get("show_specifications")).strip() != "":
|
|
prod.show_specifications = show_specs_val
|
|
if mapped_row.get("is_service_booking") is not None and str(mapped_row.get("is_service_booking")).strip() != "":
|
|
prod.is_service_booking = is_service_val
|
|
if product_cache is not None:
|
|
product_cache[parent_slug] = prod
|
|
|
|
# 4. Create or Update Variant
|
|
if not variant:
|
|
v_id = str(ulid.ULID())
|
|
variant = ProductVariant(
|
|
variant_id=v_id,
|
|
product_id=prod.product_id,
|
|
sku=sku,
|
|
price=price_val,
|
|
cost_price=cost_price_val,
|
|
status="active"
|
|
)
|
|
db_session.add(variant)
|
|
|
|
snapshot = MigrationSnapshot(
|
|
batch_id=batch_id,
|
|
entity_table="product_variants",
|
|
entity_id=variant.variant_id,
|
|
action_taken="INSERTED",
|
|
previous_state=None
|
|
)
|
|
db_session.add(snapshot)
|
|
|
|
else:
|
|
prev_state = {
|
|
"price": float(variant.price or 0.0),
|
|
"cost_price": float(variant.cost_price or 0.0),
|
|
"sku": str(variant.sku)
|
|
}
|
|
variant.price = price_val
|
|
if cost_price_val > 0:
|
|
variant.cost_price = cost_price_val
|
|
variant.product_id = prod.product_id
|
|
|
|
snapshot = MigrationSnapshot(
|
|
batch_id=batch_id,
|
|
entity_table="product_variants",
|
|
entity_id=variant.variant_id,
|
|
action_taken="UPDATED",
|
|
previous_state=prev_state
|
|
)
|
|
db_session.add(snapshot)
|
|
|
|
# 5. Dynamic Master Attributes (attribute_types) & Variant Attributes (variant_attributes)
|
|
all_attributes: Dict[str, Any] = {}
|
|
if "attributes" in mapped_row and isinstance(mapped_row["attributes"], dict):
|
|
all_attributes.update(mapped_row["attributes"])
|
|
|
|
if mapped_row.get("media_key"):
|
|
all_attributes["media_key"] = mapped_row["media_key"]
|
|
if mapped_row.get("parent_media_key"):
|
|
all_attributes["parent_media_key"] = mapped_row["parent_media_key"]
|
|
|
|
EXCLUDED_DYNAMIC_KEYS = (
|
|
"sku", "name", "parent_name", "price", "cost_price", "stock", "brand",
|
|
"category", "parent_category", "is_parent_feature", "barcode", "description",
|
|
"warranty_type", "warranty_summary", "seo_title", "seo_description",
|
|
"meta_keywords", "attributes", "media_key", "parent_media_key", "device_series",
|
|
"device_model", "device_type", "show_specifications", "is_service_booking",
|
|
"service_category", "store_locations", "is_active"
|
|
)
|
|
|
|
for key, val in mapped_row.items():
|
|
if key not in EXCLUDED_DYNAMIC_KEYS:
|
|
if val:
|
|
all_attributes[key] = val
|
|
|
|
# Auto-infer color from variant name or media_key if not explicitly provided
|
|
if not mapped_row.get("color") and not all_attributes.get("color"):
|
|
var_name = str(mapped_row.get("name") or "").strip()
|
|
if " - " in var_name:
|
|
suffix = var_name.split(" - ", 1)[1].strip()
|
|
if suffix and suffix.lower() not in {"n/a", "na", "none", "null", "-", "true", "false", "undefined"}:
|
|
all_attributes["color"] = suffix
|
|
elif mapped_row.get("media_key"):
|
|
mk = mapped_row["media_key"].lower().replace("_", "-")
|
|
for c_cand in ["forest-blue", "hot-pink", "marshmallow-blue", "black", "white", "blue", "pink", "green", "red", "gold", "silver", "grey", "gray", "purple"]:
|
|
if mk.endswith(c_cand) or f"-{c_cand}-" in mk:
|
|
all_attributes["color"] = c_cand.replace("-", " ").title()
|
|
break
|
|
|
|
# Filter meaningful attributes for variant display & mapping (ignore placeholders & boolean flags)
|
|
# NOTE: media_key/parent_media_key are intentionally excluded here to keep specs display clean,
|
|
# but they MUST be saved separately below so the MEDIA_LINK phase can match them.
|
|
PLACEHOLDER_VALUES = {"n/a", "na", "none", "null", "-", "true", "false", "undefined"}
|
|
clean_attributes = {
|
|
k: str(v).strip() for k, v in all_attributes.items()
|
|
if v is not None and str(v).strip() and str(v).strip().lower() not in PLACEHOLDER_VALUES
|
|
and k not in ("media_key", "parent_media_key", "is_active", "is_service_booking")
|
|
}
|
|
|
|
if all_attributes:
|
|
from sqlalchemy.orm.attributes import flag_modified
|
|
|
|
# Clear previous variant_attributes links for this variant
|
|
db_session.query(VariantAttribute).filter(VariantAttribute.variant_id == variant.variant_id).delete()
|
|
|
|
for attr_name, attr_val in clean_attributes.items():
|
|
attr_code = attr_name.lower().strip().replace(" ", "_").replace("-", "_")
|
|
|
|
# Find or Create AttributeType in Master Attributes (attribute_types)
|
|
attr_type = None
|
|
if attr_type_cache is not None and attr_code in attr_type_cache:
|
|
attr_type = attr_type_cache[attr_code]
|
|
else:
|
|
attr_type = db_session.query(AttributeType).filter(AttributeType.code == attr_code).first()
|
|
if not attr_type:
|
|
attr_type = db_session.query(AttributeType).filter(AttributeType.name == attr_name).first()
|
|
|
|
if not attr_type:
|
|
at_id = str(ulid.ULID())
|
|
attr_type = AttributeType(
|
|
attribute_id=at_id,
|
|
name=attr_name,
|
|
code=attr_code,
|
|
status="active",
|
|
preset_values=[]
|
|
)
|
|
db_session.add(attr_type)
|
|
if attr_type_cache is not None:
|
|
attr_type_cache[attr_code] = attr_type
|
|
|
|
# Split on comma for multi-value attributes
|
|
sub_values = [s.strip() for s in str(attr_val).split(",") if s.strip()]
|
|
valid_sub_values = [sv for sv in sub_values if sv.lower() not in PLACEHOLDER_VALUES]
|
|
|
|
seen_variant_values = set()
|
|
for sub_v in valid_sub_values:
|
|
if sub_v.lower() in seen_variant_values:
|
|
continue
|
|
seen_variant_values.add(sub_v.lower())
|
|
|
|
va_id = str(ulid.ULID())
|
|
var_attr = VariantAttribute(
|
|
id=va_id,
|
|
variant_id=variant.variant_id,
|
|
attribute_id=attr_type.attribute_id,
|
|
attribute_value=sub_v[:2000]
|
|
)
|
|
db_session.add(var_attr)
|
|
|
|
# Auto-sync distinct values to Master Attribute preset_values (case-insensitive deduplication)
|
|
current_presets = list(attr_type.preset_values or [])
|
|
existing_lower = {str(p).strip().lower() for p in current_presets}
|
|
modified_presets = False
|
|
for sub_v in valid_sub_values:
|
|
if sub_v.lower() not in existing_lower:
|
|
current_presets.append(sub_v)
|
|
existing_lower.add(sub_v.lower())
|
|
modified_presets = True
|
|
|
|
if modified_presets:
|
|
attr_type.preset_values = current_presets
|
|
flag_modified(attr_type, "preset_values")
|
|
|
|
# 6. Attach Variant Images via Smart Canonical Media Key Resolver (Supporting Comma-Separated Keys)
|
|
raw_media_key = mapped_row.get("media_key") or mapped_row.get("parent_media_key")
|
|
|
|
# Tier 2 Fallback: Match by SKU if no media_key column is in Excel
|
|
if not raw_media_key:
|
|
raw_media_key = mapped_row.get("sku")
|
|
|
|
# Tier 3 Fallback: Match by Product Name slug
|
|
if not raw_media_key:
|
|
raw_media_key = mapped_row.get("name") or mapped_row.get("parent_name")
|
|
|
|
if raw_media_key:
|
|
mk_tokens = [k.strip() for k in str(raw_media_key).split(",") if k.strip()]
|
|
if not mk_tokens and str(raw_media_key).strip():
|
|
mk_tokens = [str(raw_media_key).strip()]
|
|
|
|
all_matched_assets = []
|
|
seen_asset_ids = set()
|
|
|
|
for token_key in mk_tokens:
|
|
c_media_key = MediaResolver.canonicalize_media_key(token_key)
|
|
if not c_media_key:
|
|
continue
|
|
|
|
m_group = None
|
|
if media_group_cache is not None and c_media_key in media_group_cache:
|
|
m_group = media_group_cache[c_media_key]
|
|
else:
|
|
m_group = db_session.query(MediaGroup).filter(MediaGroup.media_key == c_media_key).first()
|
|
if not m_group and mapped_row.get("parent_media_key"):
|
|
p_tokens = [pk.strip() for pk in str(mapped_row.get("parent_media_key")).split(",") if pk.strip()]
|
|
for p_tok in p_tokens:
|
|
p_key = MediaResolver.canonicalize_media_key(p_tok)
|
|
if p_key:
|
|
m_group = db_session.query(MediaGroup).filter(MediaGroup.media_key == p_key).first()
|
|
if m_group:
|
|
break
|
|
if media_group_cache is not None:
|
|
media_group_cache[c_media_key] = m_group
|
|
|
|
if m_group and m_group.media_assets:
|
|
for asset in m_group.media_assets:
|
|
if asset.id not in seen_asset_ids:
|
|
seen_asset_ids.add(asset.id)
|
|
all_matched_assets.append(asset)
|
|
|
|
if all_matched_assets:
|
|
db_session.query(VariantImage).filter(VariantImage.variant_id == variant.variant_id).delete()
|
|
for idx, asset in enumerate(all_matched_assets):
|
|
vi_id = str(ulid.ULID())
|
|
var_img = VariantImage(
|
|
image_id=vi_id,
|
|
variant_id=variant.variant_id,
|
|
image_url=asset.cdn_url,
|
|
sort_order=idx,
|
|
is_primary=(idx == 0)
|
|
)
|
|
db_session.add(var_img)
|
|
# 7. Update inventory ledger stock level
|
|
stock_val = mapped_row.get("stock")
|
|
if stock_val is not None and str(stock_val).strip() != "":
|
|
try:
|
|
stock_qty = int(float(stock_val))
|
|
if stock_qty >= 0:
|
|
from app.services.InventoryService import apply_stock_target
|
|
apply_stock_target(
|
|
variant_id=variant.variant_id,
|
|
target_qty=stock_qty,
|
|
db=db_session,
|
|
notes=f"Imported stock level via migration batch {batch_id}",
|
|
commit=False
|
|
)
|
|
except (ValueError, TypeError):
|
|
pass
|
|
|
|
# 8. Service Booking Mapping (auto-create RepairService & RepairVariant)
|
|
if prod.is_service_booking and (service_category_val or prod.mapped_service_category_id):
|
|
st_obj = None
|
|
if service_category_val:
|
|
sc_clean = str(service_category_val).strip()
|
|
st_obj = db_session.query(ServiceType).filter(
|
|
(ServiceType.service_type_id == sc_clean) |
|
|
(ServiceType.name == sc_clean) |
|
|
(ServiceType.slug == sc_clean.lower().replace(" ", "-"))
|
|
).first()
|
|
if not st_obj:
|
|
st_obj = db_session.query(ServiceType).filter(ServiceType.name.ilike(f"%{sc_clean}%")).first()
|
|
if not st_obj:
|
|
st_id = str(ulid.ULID())
|
|
st_slug = sc_clean.lower().replace(" ", "-")
|
|
st_obj = ServiceType(
|
|
service_type_id=st_id,
|
|
name=sc_clean,
|
|
slug=st_slug,
|
|
description=f"Auto-created service category for {sc_clean}",
|
|
is_active=True
|
|
)
|
|
db_session.add(st_obj)
|
|
db_session.flush()
|
|
|
|
if not st_obj and prod.mapped_service_category_id:
|
|
st_obj = db_session.query(ServiceType).filter(ServiceType.service_type_id == prod.mapped_service_category_id).first()
|
|
|
|
if st_obj:
|
|
prod.mapped_service_category_id = st_obj.service_type_id
|
|
|
|
if prod.device_model_id:
|
|
repair_service = db_session.query(RepairService).filter(
|
|
RepairService.model_id == prod.device_model_id,
|
|
RepairService.service_type_id == st_obj.service_type_id
|
|
).first()
|
|
|
|
if not repair_service:
|
|
mod = db_session.query(DeviceModel).filter(DeviceModel.model_id == prod.device_model_id).first()
|
|
rs_slug = f"{st_obj.slug}-{mod.slug}" if mod else f"{st_obj.slug}-{prod.device_model_id}"
|
|
rs_full_path = f"/{mod.full_path.strip('/')}/{st_obj.slug}" if mod and mod.full_path else f"/{rs_slug}"
|
|
repair_service = RepairService(
|
|
repair_service_id=str(ulid.ULID()),
|
|
model_id=prod.device_model_id,
|
|
service_type_id=st_obj.service_type_id,
|
|
slug=rs_slug,
|
|
full_path=rs_full_path,
|
|
description=f"Auto-mapped service from product {prod.name}"
|
|
)
|
|
db_session.add(repair_service)
|
|
db_session.flush()
|
|
|
|
# Prefer material/quality attribute, or meaningful attribute summary, or variant/product name
|
|
material_val = clean_attributes.get("material") or clean_attributes.get("quality")
|
|
if material_val:
|
|
rv_name = material_val
|
|
elif clean_attributes:
|
|
rv_name = " | ".join(clean_attributes.values())
|
|
elif raw_name and raw_name.strip():
|
|
rv_name = raw_name.strip()
|
|
else:
|
|
rv_name = variant.sku or "Standard Quality"
|
|
|
|
rv = db_session.query(RepairVariant).filter(
|
|
RepairVariant.repair_service_id == repair_service.repair_service_id,
|
|
RepairVariant.name == rv_name[:100]
|
|
).first()
|
|
|
|
if not rv:
|
|
rv = RepairVariant(
|
|
variant_id=str(ulid.ULID()),
|
|
repair_service_id=repair_service.repair_service_id,
|
|
name=rv_name[:100],
|
|
price=price_val,
|
|
cost=cost_price_val,
|
|
duration_minutes=45,
|
|
warranty_days=90,
|
|
status="active"
|
|
)
|
|
db_session.add(rv)
|
|
db_session.flush()
|
|
else:
|
|
rv.price = price_val
|
|
if cost_price_val > 0:
|
|
rv.cost = cost_price_val
|
|
|
|
# 9. Store Locations & Multi-Branch Inventory Mapping
|
|
store_locations_raw = mapped_row.get("store_locations")
|
|
if store_locations_raw and str(store_locations_raw).strip() != "":
|
|
loc_tokens = [tok.strip() for tok in re.split(r'[;,]+', str(store_locations_raw)) if tok.strip()]
|
|
for token in loc_tokens:
|
|
parts = [p.strip() for p in token.split(":")]
|
|
loc_code = parts[0]
|
|
loc_qty = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else (int(float(stock_val)) if stock_val is not None and str(stock_val).strip() != "" else 0)
|
|
loc_selling = Decimal(str(parts[2])) if len(parts) > 2 else Decimal(str(price_val))
|
|
loc_cost = Decimal(str(cost_price_val))
|
|
|
|
crm_loc = db_session.query(CrmLocation).filter(
|
|
(CrmLocation.code == loc_code) |
|
|
(CrmLocation.name == loc_code) |
|
|
(CrmLocation.location_id == loc_code)
|
|
).first()
|
|
|
|
if crm_loc:
|
|
prod_loc = db_session.query(ProductLocation).filter(
|
|
ProductLocation.product_id == prod.product_id,
|
|
ProductLocation.location_id == crm_loc.location_id
|
|
).first()
|
|
|
|
if not prod_loc:
|
|
prod_loc = ProductLocation(
|
|
mapping_id=str(ulid.ULID()),
|
|
tenant_id=crm_loc.tenant_id or "tenant_default",
|
|
product_id=prod.product_id,
|
|
location_id=crm_loc.location_id,
|
|
stock_quantity=loc_qty,
|
|
reserved_quantity=0,
|
|
reorder_level=5,
|
|
cost_price=loc_cost,
|
|
selling_price=loc_selling,
|
|
is_active=True
|
|
)
|
|
db_session.add(prod_loc)
|
|
else:
|
|
prod_loc.stock_quantity = loc_qty
|
|
prod_loc.selling_price = loc_selling
|
|
prod_loc.cost_price = loc_cost
|
|
prod_loc.is_active = True
|
|
|
|
sp.commit()
|
|
success_count += 1
|
|
|
|
except Exception as e:
|
|
if sp is not None:
|
|
try:
|
|
sp.rollback()
|
|
except Exception:
|
|
pass
|
|
# Evict uncommitted series and models from cache to prevent cascading foreign key failures
|
|
if series_cache is not None:
|
|
for sk in new_series_keys:
|
|
series_cache.pop(sk, None)
|
|
if model_cache is not None:
|
|
for mk in new_model_keys:
|
|
model_cache.pop(mk, None)
|
|
fail_count += 1
|
|
runtime_errors.append({
|
|
"row_number": row_number,
|
|
"sku": sku,
|
|
"severity": "ERROR",
|
|
"field_name": "execution",
|
|
"error_message": f"Database transaction error: {str(e)}",
|
|
"suggested_fix": "Check database constraints and field types"
|
|
})
|
|
|
|
db_session.commit()
|
|
return success_count, fail_count, runtime_errors
|