feat(migration): add resilient CSV encoding, auto-SKU generation, and fuzzy media key matching

This commit is contained in:
adithiyan 2026-10-30 10:16:19 +05:30
parent af04fd2d6a
commit 3d05d0533b
6 changed files with 149 additions and 9 deletions

View file

@ -87,4 +87,49 @@ class ColumnMapper:
if attributes:
mapped_row["attributes"] = attributes
if not mapped_row.get("sku") or not str(mapped_row.get("sku")).strip():
mapped_row["sku"] = cls.generate_sku(mapped_row)
return mapped_row
@classmethod
def generate_sku(cls, mapped_row: Dict[str, Any], suffix: str = "") -> str:
"""
Automatically generates a clean uppercase SKU from name, device_type, and device_model.
Format: [BRAND]-[MODEL_SLUG]-[NAME_SLUG]
"""
import re
device_type = str(mapped_row.get("device_type") or (mapped_row.get("attributes") or {}).get("device_type") or "").strip().upper()
device_model = str(mapped_row.get("device_model") or (mapped_row.get("attributes") or {}).get("device_model") or "").strip()
brand = str(mapped_row.get("brand") or (mapped_row.get("attributes") or {}).get("brand") or "").strip()
name = str(mapped_row.get("name") or mapped_row.get("parent_name") or "ITEM").strip()
def clean_slug(text: str, max_words: int = 5) -> str:
text = re.sub(r'[^a-zA-Z0-9\s]+', ' ', text)
words = [w.upper() for w in text.split() if w]
return "-".join(words[:max_words])
parts = []
if brand:
parts.append(clean_slug(brand, 1))
if device_model:
model_slug = clean_slug(device_model, 4)
if parts and model_slug.startswith(parts[0] + "-"):
model_slug = model_slug[len(parts[0]) + 1:]
parts.append(model_slug)
elif device_type:
parts.append(clean_slug(device_type, 1))
name_slug = clean_slug(name, 4)
if name_slug:
parts.append(name_slug)
sku_base = "-".join([p for p in parts if p]).strip("-")
if not sku_base:
sku_base = "IFX-ITEM"
if suffix:
sku_base = f"{sku_base}-{suffix}"
return sku_base

View file

@ -10,6 +10,26 @@ class DataFileParser:
Supports streaming generation of rows to minimize RAM usage on large imports (100,000+ rows).
"""
@classmethod
def _detect_csv_encoding(cls, file_path: str) -> str:
"""
Detects appropriate encoding for CSV files, handling Excel CP1252/Windows-1252 gracefully.
"""
try:
with open(file_path, "rb") as bf:
sample = bf.read(131072)
try:
sample.decode("utf-8")
return "utf-8-sig"
except UnicodeDecodeError:
try:
sample.decode("cp1252")
return "cp1252"
except UnicodeDecodeError:
return "latin1"
except Exception:
return "utf-8-sig"
@classmethod
def get_headers(cls, file_path: str, file_format: str) -> List[str]:
"""
@ -17,7 +37,8 @@ class DataFileParser:
"""
fmt = file_format.upper()
if fmt == "CSV":
with open(file_path, "r", encoding="utf-8-sig") as f:
enc = cls._detect_csv_encoding(file_path)
with open(file_path, "r", encoding=enc, errors="replace") as f:
reader = csv.reader(f)
for row in reader:
return [c.strip() for c in row if c is not None]
@ -37,7 +58,8 @@ class DataFileParser:
"""
fmt = file_format.upper()
if fmt == "CSV":
with open(file_path, "r", encoding="utf-8-sig") as f:
enc = cls._detect_csv_encoding(file_path)
with open(file_path, "r", encoding=enc, errors="replace") as f:
reader = csv.DictReader(f)
row_idx = 2 # Row 1 is headers
for row in reader:

View file

@ -137,8 +137,7 @@ class MediaResolver:
if parsed_key:
return parsed_key
# 3. If FLAT mode specified but no '__' in filename, try slugifying filename before extension
if media_structure == "FLAT":
# 3. Slugify filename before extension (works for FLAT and flat-archive AUTO)
parsed_key = cls.canonicalize_media_key(name_without_ext)
if parsed_key:
return parsed_key
@ -247,8 +246,20 @@ class MediaResolver:
pass
rel_path = os.path.relpath(full_path, extract_to_dir)
item_media_key = dir_media_key
if item_media_key == "general-media":
item_media_key = cls.resolve_entry_media_key(rel_path, wrapper_prefix=wrapper_prefix, media_structure="FLAT")
if item_media_key not in media_groups:
media_groups[item_media_key] = {
"media_key": item_media_key,
"source_type": "FLAT_FILE",
"brand_name": brand,
"model_name": model,
"variant_tag": variant
}
media_files.append({
"media_key": dir_media_key,
"media_key": item_media_key,
"original_filename": filename,
"relative_path": rel_path,
"local_path": full_path,

View file

@ -378,8 +378,17 @@ class MigrationWorker:
brand_cache, category_cache, series_cache, model_cache, product_cache, attr_type_cache, media_group_cache = {}, {}, {}, {}, {}, {}, {}
seen_skus: Dict[str, int] = {}
for row_idx, raw_row in DataFileParser.stream_rows(file_path, ext):
mapped_row = ColumnMapper.apply_mapping(raw_row, column_maps)
cur_sku = mapped_row.get("sku")
if cur_sku:
if cur_sku in seen_skus:
seen_skus[cur_sku] += 1
mapped_row["sku"] = f"{cur_sku}-{seen_skus[cur_sku]:02d}"
else:
seen_skus[cur_sku] = 1
current_chunk.append((row_idx, mapped_row))
if len(current_chunk) >= chunk_size:
@ -744,6 +753,19 @@ class MigrationWorker:
brands = {b.brand_id: b.name for b in db.query(Brand).all()}
products = {p.product_id: p for p in db.query(Product).all()}
media_attr_types = db.query(AttributeType).filter(AttributeType.code.in_(["media_key", "parent_media_key"])).all()
media_attr_ids = {at.attribute_id: at.code for at in media_attr_types}
variant_media_keys = {}
variant_parent_media_keys = {}
if media_attr_ids:
for va in db.query(VariantAttribute).filter(VariantAttribute.attribute_id.in_(list(media_attr_ids.keys()))).all():
code = media_attr_ids.get(va.attribute_id)
if code == "media_key":
variant_media_keys[va.variant_id] = va.attribute_value
elif code == "parent_media_key":
variant_parent_media_keys[va.variant_id] = va.attribute_value
variant_colors = {}
if color_attr_id:
for va in db.query(VariantAttribute).filter(VariantAttribute.attribute_id == color_attr_id).all():
@ -754,18 +776,49 @@ class MigrationWorker:
if va.attribute_value:
va_by_value.setdefault(va.attribute_value, []).append(va)
import re
for v in db.query(ProductVariant).all():
p = products.get(v.product_id)
if not p: continue
color = variant_colors.get(v.variant_id, '')
brand_name = brands.get(p.brand_id, '')
# Priority 1: Direct media keys from CSV attributes
raw_mk = variant_media_keys.get(v.variant_id)
raw_pmk = variant_parent_media_keys.get(v.variant_id)
c_mk = MediaResolver.canonicalize_media_key(raw_mk) if raw_mk else None
c_pmk = MediaResolver.canonicalize_media_key(raw_pmk) if raw_pmk else None
# Priority 2: Synthetic name/slug keys
short_slug = p.slug.split('-for-')[0] if '-for-' in p.slug else p.slug
key3 = MediaResolver.canonicalize_media_key(f'{short_slug} {color}')
key1 = MediaResolver.canonicalize_media_key(f'{brand_name} {p.name} {color}')
key2 = MediaResolver.canonicalize_media_key(f'{p.slug} {color}')
mg = mg_map.get(key3) or mg_map.get(key1) or mg_map.get(key2)
mg = None
for candidate in [c_mk, c_pmk, raw_mk, raw_pmk, key3, key1, key2]:
if candidate and candidate in mg_map:
mg = mg_map[candidate]
break
# Priority 3: Fuzzy token-similarity matching if exact key has minor variance
if not mg and (c_mk or c_pmk or p.name):
target_str = c_mk or c_pmk or f"{p.name} {color}"
target_tokens = set(re.findall(r'[a-zA-Z0-9]+', target_str.lower()))
if target_tokens:
best_cand, best_score = None, 0.0
for cand_key in mg_map.keys():
cand_tokens = set(re.findall(r'[a-zA-Z0-9]+', cand_key.lower()))
if not cand_tokens: continue
overlap = len(target_tokens & cand_tokens)
score = overlap / max(len(target_tokens), len(cand_tokens))
if score > best_score:
best_score = score
best_cand = cand_key
if best_score >= 0.55: # High confidence token overlap
mg = mg_map[best_cand]
if mg and mg.media_assets:
if v.variant_id not in existing_vi_variant_ids:
for idx, asset in enumerate(mg.media_assets):

View file

@ -59,6 +59,10 @@ class UpsertExecutor:
for row_number, mapped_row in mapped_rows:
sku = mapped_row.get("sku")
if not sku:
from app.services.migration_engine.column_mapper import ColumnMapper
sku = ColumnMapper.generate_sku(mapped_row)
mapped_row["sku"] = sku
if not sku:
fail_count += 1
runtime_errors.append({

View file

@ -35,6 +35,11 @@ class ValidationEngine:
media_key = mapped_row.get("media_key")
# 1. Required Field Checks
if not sku:
from app.services.migration_engine.column_mapper import ColumnMapper
sku = ColumnMapper.generate_sku(mapped_row)
mapped_row["sku"] = sku
if not sku:
errors.append({
"row_number": row_number,