feat(migration): add resilient CSV encoding, auto-SKU generation, and fuzzy media key matching
This commit is contained in:
parent
af04fd2d6a
commit
3d05d0533b
6 changed files with 149 additions and 9 deletions
|
|
@ -87,4 +87,49 @@ class ColumnMapper:
|
|||
|
||||
if attributes:
|
||||
mapped_row["attributes"] = attributes
|
||||
|
||||
if not mapped_row.get("sku") or not str(mapped_row.get("sku")).strip():
|
||||
mapped_row["sku"] = cls.generate_sku(mapped_row)
|
||||
|
||||
return mapped_row
|
||||
|
||||
@classmethod
|
||||
def generate_sku(cls, mapped_row: Dict[str, Any], suffix: str = "") -> str:
|
||||
"""
|
||||
Automatically generates a clean uppercase SKU from name, device_type, and device_model.
|
||||
Format: [BRAND]-[MODEL_SLUG]-[NAME_SLUG]
|
||||
"""
|
||||
import re
|
||||
device_type = str(mapped_row.get("device_type") or (mapped_row.get("attributes") or {}).get("device_type") or "").strip().upper()
|
||||
device_model = str(mapped_row.get("device_model") or (mapped_row.get("attributes") or {}).get("device_model") or "").strip()
|
||||
brand = str(mapped_row.get("brand") or (mapped_row.get("attributes") or {}).get("brand") or "").strip()
|
||||
name = str(mapped_row.get("name") or mapped_row.get("parent_name") or "ITEM").strip()
|
||||
|
||||
def clean_slug(text: str, max_words: int = 5) -> str:
|
||||
text = re.sub(r'[^a-zA-Z0-9\s]+', ' ', text)
|
||||
words = [w.upper() for w in text.split() if w]
|
||||
return "-".join(words[:max_words])
|
||||
|
||||
parts = []
|
||||
if brand:
|
||||
parts.append(clean_slug(brand, 1))
|
||||
if device_model:
|
||||
model_slug = clean_slug(device_model, 4)
|
||||
if parts and model_slug.startswith(parts[0] + "-"):
|
||||
model_slug = model_slug[len(parts[0]) + 1:]
|
||||
parts.append(model_slug)
|
||||
elif device_type:
|
||||
parts.append(clean_slug(device_type, 1))
|
||||
|
||||
name_slug = clean_slug(name, 4)
|
||||
if name_slug:
|
||||
parts.append(name_slug)
|
||||
|
||||
sku_base = "-".join([p for p in parts if p]).strip("-")
|
||||
if not sku_base:
|
||||
sku_base = "IFX-ITEM"
|
||||
|
||||
if suffix:
|
||||
sku_base = f"{sku_base}-{suffix}"
|
||||
|
||||
return sku_base
|
||||
|
|
|
|||
|
|
@ -10,6 +10,26 @@ class DataFileParser:
|
|||
Supports streaming generation of rows to minimize RAM usage on large imports (100,000+ rows).
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def _detect_csv_encoding(cls, file_path: str) -> str:
|
||||
"""
|
||||
Detects appropriate encoding for CSV files, handling Excel CP1252/Windows-1252 gracefully.
|
||||
"""
|
||||
try:
|
||||
with open(file_path, "rb") as bf:
|
||||
sample = bf.read(131072)
|
||||
try:
|
||||
sample.decode("utf-8")
|
||||
return "utf-8-sig"
|
||||
except UnicodeDecodeError:
|
||||
try:
|
||||
sample.decode("cp1252")
|
||||
return "cp1252"
|
||||
except UnicodeDecodeError:
|
||||
return "latin1"
|
||||
except Exception:
|
||||
return "utf-8-sig"
|
||||
|
||||
@classmethod
|
||||
def get_headers(cls, file_path: str, file_format: str) -> List[str]:
|
||||
"""
|
||||
|
|
@ -17,7 +37,8 @@ class DataFileParser:
|
|||
"""
|
||||
fmt = file_format.upper()
|
||||
if fmt == "CSV":
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
enc = cls._detect_csv_encoding(file_path)
|
||||
with open(file_path, "r", encoding=enc, errors="replace") as f:
|
||||
reader = csv.reader(f)
|
||||
for row in reader:
|
||||
return [c.strip() for c in row if c is not None]
|
||||
|
|
@ -37,7 +58,8 @@ class DataFileParser:
|
|||
"""
|
||||
fmt = file_format.upper()
|
||||
if fmt == "CSV":
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
enc = cls._detect_csv_encoding(file_path)
|
||||
with open(file_path, "r", encoding=enc, errors="replace") as f:
|
||||
reader = csv.DictReader(f)
|
||||
row_idx = 2 # Row 1 is headers
|
||||
for row in reader:
|
||||
|
|
|
|||
|
|
@ -137,8 +137,7 @@ class MediaResolver:
|
|||
if parsed_key:
|
||||
return parsed_key
|
||||
|
||||
# 3. If FLAT mode specified but no '__' in filename, try slugifying filename before extension
|
||||
if media_structure == "FLAT":
|
||||
# 3. Slugify filename before extension (works for FLAT and flat-archive AUTO)
|
||||
parsed_key = cls.canonicalize_media_key(name_without_ext)
|
||||
if parsed_key:
|
||||
return parsed_key
|
||||
|
|
@ -247,8 +246,20 @@ class MediaResolver:
|
|||
pass
|
||||
|
||||
rel_path = os.path.relpath(full_path, extract_to_dir)
|
||||
item_media_key = dir_media_key
|
||||
if item_media_key == "general-media":
|
||||
item_media_key = cls.resolve_entry_media_key(rel_path, wrapper_prefix=wrapper_prefix, media_structure="FLAT")
|
||||
if item_media_key not in media_groups:
|
||||
media_groups[item_media_key] = {
|
||||
"media_key": item_media_key,
|
||||
"source_type": "FLAT_FILE",
|
||||
"brand_name": brand,
|
||||
"model_name": model,
|
||||
"variant_tag": variant
|
||||
}
|
||||
|
||||
media_files.append({
|
||||
"media_key": dir_media_key,
|
||||
"media_key": item_media_key,
|
||||
"original_filename": filename,
|
||||
"relative_path": rel_path,
|
||||
"local_path": full_path,
|
||||
|
|
|
|||
|
|
@ -378,8 +378,17 @@ class MigrationWorker:
|
|||
|
||||
brand_cache, category_cache, series_cache, model_cache, product_cache, attr_type_cache, media_group_cache = {}, {}, {}, {}, {}, {}, {}
|
||||
|
||||
seen_skus: Dict[str, int] = {}
|
||||
|
||||
for row_idx, raw_row in DataFileParser.stream_rows(file_path, ext):
|
||||
mapped_row = ColumnMapper.apply_mapping(raw_row, column_maps)
|
||||
cur_sku = mapped_row.get("sku")
|
||||
if cur_sku:
|
||||
if cur_sku in seen_skus:
|
||||
seen_skus[cur_sku] += 1
|
||||
mapped_row["sku"] = f"{cur_sku}-{seen_skus[cur_sku]:02d}"
|
||||
else:
|
||||
seen_skus[cur_sku] = 1
|
||||
current_chunk.append((row_idx, mapped_row))
|
||||
|
||||
if len(current_chunk) >= chunk_size:
|
||||
|
|
@ -744,6 +753,19 @@ class MigrationWorker:
|
|||
brands = {b.brand_id: b.name for b in db.query(Brand).all()}
|
||||
products = {p.product_id: p for p in db.query(Product).all()}
|
||||
|
||||
media_attr_types = db.query(AttributeType).filter(AttributeType.code.in_(["media_key", "parent_media_key"])).all()
|
||||
media_attr_ids = {at.attribute_id: at.code for at in media_attr_types}
|
||||
|
||||
variant_media_keys = {}
|
||||
variant_parent_media_keys = {}
|
||||
if media_attr_ids:
|
||||
for va in db.query(VariantAttribute).filter(VariantAttribute.attribute_id.in_(list(media_attr_ids.keys()))).all():
|
||||
code = media_attr_ids.get(va.attribute_id)
|
||||
if code == "media_key":
|
||||
variant_media_keys[va.variant_id] = va.attribute_value
|
||||
elif code == "parent_media_key":
|
||||
variant_parent_media_keys[va.variant_id] = va.attribute_value
|
||||
|
||||
variant_colors = {}
|
||||
if color_attr_id:
|
||||
for va in db.query(VariantAttribute).filter(VariantAttribute.attribute_id == color_attr_id).all():
|
||||
|
|
@ -754,18 +776,49 @@ class MigrationWorker:
|
|||
if va.attribute_value:
|
||||
va_by_value.setdefault(va.attribute_value, []).append(va)
|
||||
|
||||
import re
|
||||
|
||||
for v in db.query(ProductVariant).all():
|
||||
p = products.get(v.product_id)
|
||||
if not p: continue
|
||||
color = variant_colors.get(v.variant_id, '')
|
||||
brand_name = brands.get(p.brand_id, '')
|
||||
|
||||
# Priority 1: Direct media keys from CSV attributes
|
||||
raw_mk = variant_media_keys.get(v.variant_id)
|
||||
raw_pmk = variant_parent_media_keys.get(v.variant_id)
|
||||
c_mk = MediaResolver.canonicalize_media_key(raw_mk) if raw_mk else None
|
||||
c_pmk = MediaResolver.canonicalize_media_key(raw_pmk) if raw_pmk else None
|
||||
|
||||
# Priority 2: Synthetic name/slug keys
|
||||
short_slug = p.slug.split('-for-')[0] if '-for-' in p.slug else p.slug
|
||||
key3 = MediaResolver.canonicalize_media_key(f'{short_slug} {color}')
|
||||
key1 = MediaResolver.canonicalize_media_key(f'{brand_name} {p.name} {color}')
|
||||
key2 = MediaResolver.canonicalize_media_key(f'{p.slug} {color}')
|
||||
|
||||
mg = mg_map.get(key3) or mg_map.get(key1) or mg_map.get(key2)
|
||||
mg = None
|
||||
for candidate in [c_mk, c_pmk, raw_mk, raw_pmk, key3, key1, key2]:
|
||||
if candidate and candidate in mg_map:
|
||||
mg = mg_map[candidate]
|
||||
break
|
||||
|
||||
# Priority 3: Fuzzy token-similarity matching if exact key has minor variance
|
||||
if not mg and (c_mk or c_pmk or p.name):
|
||||
target_str = c_mk or c_pmk or f"{p.name} {color}"
|
||||
target_tokens = set(re.findall(r'[a-zA-Z0-9]+', target_str.lower()))
|
||||
if target_tokens:
|
||||
best_cand, best_score = None, 0.0
|
||||
for cand_key in mg_map.keys():
|
||||
cand_tokens = set(re.findall(r'[a-zA-Z0-9]+', cand_key.lower()))
|
||||
if not cand_tokens: continue
|
||||
overlap = len(target_tokens & cand_tokens)
|
||||
score = overlap / max(len(target_tokens), len(cand_tokens))
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_cand = cand_key
|
||||
if best_score >= 0.55: # High confidence token overlap
|
||||
mg = mg_map[best_cand]
|
||||
|
||||
if mg and mg.media_assets:
|
||||
if v.variant_id not in existing_vi_variant_ids:
|
||||
for idx, asset in enumerate(mg.media_assets):
|
||||
|
|
|
|||
|
|
@ -59,6 +59,10 @@ class UpsertExecutor:
|
|||
|
||||
for row_number, mapped_row in mapped_rows:
|
||||
sku = mapped_row.get("sku")
|
||||
if not sku:
|
||||
from app.services.migration_engine.column_mapper import ColumnMapper
|
||||
sku = ColumnMapper.generate_sku(mapped_row)
|
||||
mapped_row["sku"] = sku
|
||||
if not sku:
|
||||
fail_count += 1
|
||||
runtime_errors.append({
|
||||
|
|
|
|||
|
|
@ -35,6 +35,11 @@ class ValidationEngine:
|
|||
media_key = mapped_row.get("media_key")
|
||||
|
||||
# 1. Required Field Checks
|
||||
if not sku:
|
||||
from app.services.migration_engine.column_mapper import ColumnMapper
|
||||
sku = ColumnMapper.generate_sku(mapped_row)
|
||||
mapped_row["sku"] = sku
|
||||
|
||||
if not sku:
|
||||
errors.append({
|
||||
"row_number": row_number,
|
||||
|
|
|
|||
Loading…
Reference in a new issue