From 3d05d0533bb1d5303adecc6108a09e4a1683174c Mon Sep 17 00:00:00 2001 From: adithiyan Date: Fri, 30 Oct 2026 10:16:19 +0530 Subject: [PATCH] feat(migration): add resilient CSV encoding, auto-SKU generation, and fuzzy media key matching --- .../migration_engine/column_mapper.py | 45 +++++++++++++++ app/services/migration_engine/file_parsers.py | 26 ++++++++- .../migration_engine/media_resolver.py | 23 ++++++-- .../migration_engine/migration_worker.py | 55 ++++++++++++++++++- .../migration_engine/upsert_executor.py | 4 ++ .../migration_engine/validation_engine.py | 5 ++ 6 files changed, 149 insertions(+), 9 deletions(-) diff --git a/app/services/migration_engine/column_mapper.py b/app/services/migration_engine/column_mapper.py index e34bf77..f5d507d 100644 --- a/app/services/migration_engine/column_mapper.py +++ b/app/services/migration_engine/column_mapper.py @@ -87,4 +87,49 @@ class ColumnMapper: if attributes: mapped_row["attributes"] = attributes + + if not mapped_row.get("sku") or not str(mapped_row.get("sku")).strip(): + mapped_row["sku"] = cls.generate_sku(mapped_row) + return mapped_row + + @classmethod + def generate_sku(cls, mapped_row: Dict[str, Any], suffix: str = "") -> str: + """ + Automatically generates a clean uppercase SKU from name, device_type, and device_model. + Format: [BRAND]-[MODEL_SLUG]-[NAME_SLUG] + """ + import re + device_type = str(mapped_row.get("device_type") or (mapped_row.get("attributes") or {}).get("device_type") or "").strip().upper() + device_model = str(mapped_row.get("device_model") or (mapped_row.get("attributes") or {}).get("device_model") or "").strip() + brand = str(mapped_row.get("brand") or (mapped_row.get("attributes") or {}).get("brand") or "").strip() + name = str(mapped_row.get("name") or mapped_row.get("parent_name") or "ITEM").strip() + + def clean_slug(text: str, max_words: int = 5) -> str: + text = re.sub(r'[^a-zA-Z0-9\s]+', ' ', text) + words = [w.upper() for w in text.split() if w] + return "-".join(words[:max_words]) + + parts = [] + if brand: + parts.append(clean_slug(brand, 1)) + if device_model: + model_slug = clean_slug(device_model, 4) + if parts and model_slug.startswith(parts[0] + "-"): + model_slug = model_slug[len(parts[0]) + 1:] + parts.append(model_slug) + elif device_type: + parts.append(clean_slug(device_type, 1)) + + name_slug = clean_slug(name, 4) + if name_slug: + parts.append(name_slug) + + sku_base = "-".join([p for p in parts if p]).strip("-") + if not sku_base: + sku_base = "IFX-ITEM" + + if suffix: + sku_base = f"{sku_base}-{suffix}" + + return sku_base diff --git a/app/services/migration_engine/file_parsers.py b/app/services/migration_engine/file_parsers.py index d6e2e5b..ca434b7 100644 --- a/app/services/migration_engine/file_parsers.py +++ b/app/services/migration_engine/file_parsers.py @@ -10,6 +10,26 @@ class DataFileParser: Supports streaming generation of rows to minimize RAM usage on large imports (100,000+ rows). """ + @classmethod + def _detect_csv_encoding(cls, file_path: str) -> str: + """ + Detects appropriate encoding for CSV files, handling Excel CP1252/Windows-1252 gracefully. + """ + try: + with open(file_path, "rb") as bf: + sample = bf.read(131072) + try: + sample.decode("utf-8") + return "utf-8-sig" + except UnicodeDecodeError: + try: + sample.decode("cp1252") + return "cp1252" + except UnicodeDecodeError: + return "latin1" + except Exception: + return "utf-8-sig" + @classmethod def get_headers(cls, file_path: str, file_format: str) -> List[str]: """ @@ -17,7 +37,8 @@ class DataFileParser: """ fmt = file_format.upper() if fmt == "CSV": - with open(file_path, "r", encoding="utf-8-sig") as f: + enc = cls._detect_csv_encoding(file_path) + with open(file_path, "r", encoding=enc, errors="replace") as f: reader = csv.reader(f) for row in reader: return [c.strip() for c in row if c is not None] @@ -37,7 +58,8 @@ class DataFileParser: """ fmt = file_format.upper() if fmt == "CSV": - with open(file_path, "r", encoding="utf-8-sig") as f: + enc = cls._detect_csv_encoding(file_path) + with open(file_path, "r", encoding=enc, errors="replace") as f: reader = csv.DictReader(f) row_idx = 2 # Row 1 is headers for row in reader: diff --git a/app/services/migration_engine/media_resolver.py b/app/services/migration_engine/media_resolver.py index b688636..6bd1dc9 100644 --- a/app/services/migration_engine/media_resolver.py +++ b/app/services/migration_engine/media_resolver.py @@ -137,11 +137,10 @@ class MediaResolver: if parsed_key: return parsed_key - # 3. If FLAT mode specified but no '__' in filename, try slugifying filename before extension - if media_structure == "FLAT": - parsed_key = cls.canonicalize_media_key(name_without_ext) - if parsed_key: - return parsed_key + # 3. Slugify filename before extension (works for FLAT and flat-archive AUTO) + parsed_key = cls.canonicalize_media_key(name_without_ext) + if parsed_key: + return parsed_key return "general-media" @@ -247,8 +246,20 @@ class MediaResolver: pass rel_path = os.path.relpath(full_path, extract_to_dir) + item_media_key = dir_media_key + if item_media_key == "general-media": + item_media_key = cls.resolve_entry_media_key(rel_path, wrapper_prefix=wrapper_prefix, media_structure="FLAT") + if item_media_key not in media_groups: + media_groups[item_media_key] = { + "media_key": item_media_key, + "source_type": "FLAT_FILE", + "brand_name": brand, + "model_name": model, + "variant_tag": variant + } + media_files.append({ - "media_key": dir_media_key, + "media_key": item_media_key, "original_filename": filename, "relative_path": rel_path, "local_path": full_path, diff --git a/app/services/migration_engine/migration_worker.py b/app/services/migration_engine/migration_worker.py index fda5a7e..2434f98 100644 --- a/app/services/migration_engine/migration_worker.py +++ b/app/services/migration_engine/migration_worker.py @@ -378,8 +378,17 @@ class MigrationWorker: brand_cache, category_cache, series_cache, model_cache, product_cache, attr_type_cache, media_group_cache = {}, {}, {}, {}, {}, {}, {} + seen_skus: Dict[str, int] = {} + for row_idx, raw_row in DataFileParser.stream_rows(file_path, ext): mapped_row = ColumnMapper.apply_mapping(raw_row, column_maps) + cur_sku = mapped_row.get("sku") + if cur_sku: + if cur_sku in seen_skus: + seen_skus[cur_sku] += 1 + mapped_row["sku"] = f"{cur_sku}-{seen_skus[cur_sku]:02d}" + else: + seen_skus[cur_sku] = 1 current_chunk.append((row_idx, mapped_row)) if len(current_chunk) >= chunk_size: @@ -744,6 +753,19 @@ class MigrationWorker: brands = {b.brand_id: b.name for b in db.query(Brand).all()} products = {p.product_id: p for p in db.query(Product).all()} + media_attr_types = db.query(AttributeType).filter(AttributeType.code.in_(["media_key", "parent_media_key"])).all() + media_attr_ids = {at.attribute_id: at.code for at in media_attr_types} + + variant_media_keys = {} + variant_parent_media_keys = {} + if media_attr_ids: + for va in db.query(VariantAttribute).filter(VariantAttribute.attribute_id.in_(list(media_attr_ids.keys()))).all(): + code = media_attr_ids.get(va.attribute_id) + if code == "media_key": + variant_media_keys[va.variant_id] = va.attribute_value + elif code == "parent_media_key": + variant_parent_media_keys[va.variant_id] = va.attribute_value + variant_colors = {} if color_attr_id: for va in db.query(VariantAttribute).filter(VariantAttribute.attribute_id == color_attr_id).all(): @@ -754,18 +776,49 @@ class MigrationWorker: if va.attribute_value: va_by_value.setdefault(va.attribute_value, []).append(va) + import re + for v in db.query(ProductVariant).all(): p = products.get(v.product_id) if not p: continue color = variant_colors.get(v.variant_id, '') brand_name = brands.get(p.brand_id, '') + # Priority 1: Direct media keys from CSV attributes + raw_mk = variant_media_keys.get(v.variant_id) + raw_pmk = variant_parent_media_keys.get(v.variant_id) + c_mk = MediaResolver.canonicalize_media_key(raw_mk) if raw_mk else None + c_pmk = MediaResolver.canonicalize_media_key(raw_pmk) if raw_pmk else None + + # Priority 2: Synthetic name/slug keys short_slug = p.slug.split('-for-')[0] if '-for-' in p.slug else p.slug key3 = MediaResolver.canonicalize_media_key(f'{short_slug} {color}') key1 = MediaResolver.canonicalize_media_key(f'{brand_name} {p.name} {color}') key2 = MediaResolver.canonicalize_media_key(f'{p.slug} {color}') - mg = mg_map.get(key3) or mg_map.get(key1) or mg_map.get(key2) + mg = None + for candidate in [c_mk, c_pmk, raw_mk, raw_pmk, key3, key1, key2]: + if candidate and candidate in mg_map: + mg = mg_map[candidate] + break + + # Priority 3: Fuzzy token-similarity matching if exact key has minor variance + if not mg and (c_mk or c_pmk or p.name): + target_str = c_mk or c_pmk or f"{p.name} {color}" + target_tokens = set(re.findall(r'[a-zA-Z0-9]+', target_str.lower())) + if target_tokens: + best_cand, best_score = None, 0.0 + for cand_key in mg_map.keys(): + cand_tokens = set(re.findall(r'[a-zA-Z0-9]+', cand_key.lower())) + if not cand_tokens: continue + overlap = len(target_tokens & cand_tokens) + score = overlap / max(len(target_tokens), len(cand_tokens)) + if score > best_score: + best_score = score + best_cand = cand_key + if best_score >= 0.55: # High confidence token overlap + mg = mg_map[best_cand] + if mg and mg.media_assets: if v.variant_id not in existing_vi_variant_ids: for idx, asset in enumerate(mg.media_assets): diff --git a/app/services/migration_engine/upsert_executor.py b/app/services/migration_engine/upsert_executor.py index d0b48c9..2651025 100644 --- a/app/services/migration_engine/upsert_executor.py +++ b/app/services/migration_engine/upsert_executor.py @@ -59,6 +59,10 @@ class UpsertExecutor: for row_number, mapped_row in mapped_rows: sku = mapped_row.get("sku") + if not sku: + from app.services.migration_engine.column_mapper import ColumnMapper + sku = ColumnMapper.generate_sku(mapped_row) + mapped_row["sku"] = sku if not sku: fail_count += 1 runtime_errors.append({ diff --git a/app/services/migration_engine/validation_engine.py b/app/services/migration_engine/validation_engine.py index 65c553b..4c76881 100644 --- a/app/services/migration_engine/validation_engine.py +++ b/app/services/migration_engine/validation_engine.py @@ -35,6 +35,11 @@ class ValidationEngine: media_key = mapped_row.get("media_key") # 1. Required Field Checks + if not sku: + from app.services.migration_engine.column_mapper import ColumnMapper + sku = ColumnMapper.generate_sku(mapped_row) + mapped_row["sku"] = sku + if not sku: errors.append({ "row_number": row_number,