fix: resolve MEDIA_LINK NameError crash and expand VARCHAR columns to 2000

- Fix NameError in link_media_groups_to_catalog: raw_mk/raw_pmk were used
  before assignment in Priority 0 comma-split block; moved fetches to top of
  variant loop so all priority levels share the same resolved values
- Expand variant_attributes.attribute_value to String(2000) for comma-separated
  multi-image media keys
- Expand products, device_models, device_series, media_library,
  migration_media_items URL/path/name columns to String(2000)
- Update Pydantic VariantAttributeCreate.attribute_value max_length=2000
- Add rollback cache eviction for new series/model keys (fixes MySQL 1452)
- Add comma-separated media key splitting in upsert_executor for multi-image
  variant image attachment
- Add Priority 0 comma-separated MEDIA_LINK support in migration_worker
This commit is contained in:
iFixKart Dev 2026-11-06 11:34:09 +05:30
parent 7c7c474e47
commit 5dc56b2687
6 changed files with 169 additions and 88 deletions

View file

@ -10,7 +10,7 @@ class DeviceSeries(Base):
series_id = Column(String(26), primary_key=True)
brand_id = Column(String(26), ForeignKey("brands.brand_id", ondelete="SET NULL"), nullable=True)
name = Column(String(100), nullable=False)
name = Column(String(2000), nullable=False)
slug = Column(String(100), unique=True, nullable=False)
device_type = Column(String(50), nullable=True) # laptop, tablet, mobile
sort_order = Column(Integer, default=0)
@ -25,10 +25,10 @@ class DeviceModel(Base):
model_id = Column(String(26), primary_key=True)
series_id = Column(String(26), ForeignKey("device_series.series_id", ondelete="SET NULL"), nullable=True)
brand_id = Column(String(26), ForeignKey("brands.brand_id", ondelete="SET NULL"), nullable=True)
name = Column(String(100), nullable=False)
name = Column(String(2000), nullable=False)
slug = Column(String(100), unique=True, nullable=False)
device_type = Column(String(50), nullable=True) # laptop, tablet, mobile
full_path = Column(String(500), nullable=False)
full_path = Column(String(2000), nullable=False)
release_year = Column(Integer, nullable=True)
image_url = Column(String(500), nullable=True)
is_active = Column(Boolean, default=True)

View file

@ -119,16 +119,16 @@ class MediaAsset(Base):
id = Column(String(36), primary_key=True, default=generate_uuid)
media_group_id = Column(String(36), ForeignKey("media_groups.id", ondelete="CASCADE"), nullable=False)
original_filename = Column(String(255), nullable=False)
stored_filename = Column(String(255), nullable=False)
original_filename = Column(String(2000), nullable=False)
stored_filename = Column(String(2000), nullable=False)
mime_type = Column(String(50), nullable=False)
file_size_bytes = Column(BigInteger, nullable=False)
width = Column(Integer, nullable=False, default=0)
height = Column(Integer, nullable=False, default=0)
sha256_checksum = Column(String(64), unique=True, nullable=False, index=True) # Global unique SHA-256 asset
cdn_url = Column(String(512), nullable=False)
thumbnail_url = Column(String(512), nullable=False)
storage_path = Column(String(512), nullable=True)
cdn_url = Column(String(2000), nullable=False)
thumbnail_url = Column(String(2000), nullable=False)
storage_path = Column(String(2000), nullable=True)
exif_metadata = Column(JSON, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow)
@ -207,12 +207,12 @@ class MigrationMediaItem(Base):
id = Column(String(36), primary_key=True, default=generate_uuid)
job_id = Column(String(36), ForeignKey("migration_jobs.id", ondelete="CASCADE"), nullable=False, index=True)
batch_number = Column(Integer, nullable=False, default=0)
file_name = Column(String(255), nullable=False)
file_name = Column(String(2000), nullable=False)
archive_name = Column(String(255), nullable=True)
zip_entry_path = Column(String(512), nullable=True)
zip_entry_path = Column(String(2000), nullable=True)
media_key = Column(String(255), nullable=False, index=True)
sha256 = Column(String(64), nullable=True, index=True) # NOT unique (allows multiple source references to map to 1 MediaAsset)
storage_path = Column(String(512), nullable=True)
storage_path = Column(String(2000), nullable=True)
status = Column(SQLEnum(MediaItemStatusEnum), nullable=False, default=MediaItemStatusEnum.PENDING, index=True)
error = Column(Text, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow)

View file

@ -25,14 +25,14 @@ class Product(Base):
device_series_id = Column(String(26), ForeignKey("device_series.series_id", ondelete="SET NULL"), nullable=True)
device_model_id = Column(String(26), ForeignKey("device_models.model_id", ondelete="SET NULL"), nullable=True)
device_type = Column(String(50), nullable=True) # laptop, tablet, mobile
name = Column(String(255), nullable=False)
name = Column(String(2000), nullable=False)
name_normalized = Column(String(255), nullable=True, index=True)
model_sentinel = Column(String(36), nullable=False, default="GENERIC")
slug = Column(String(255), unique=True, nullable=False)
full_path = Column(String(500), nullable=False)
full_path = Column(String(2000), nullable=False)
description = Column(Text, nullable=True)
warranty_type = Column(String(100), nullable=True)
warranty_summary = Column(String(500), nullable=True)
warranty_summary = Column(String(2000), nullable=True)
seo_title = Column(Text, nullable=True)
seo_description = Column(Text, nullable=True)
meta_keywords = Column(Text, nullable=True)
@ -82,8 +82,8 @@ class ProductImage(Base):
image_id = Column(String(26), primary_key=True)
product_id = Column(String(26), ForeignKey("products.product_id", ondelete="CASCADE"), nullable=False)
image_url = Column(String(500), nullable=False)
alt_text = Column(String(255), nullable=True)
image_url = Column(String(2000), nullable=False)
alt_text = Column(String(2000), nullable=True)
sort_order = Column(Integer, default=0)
is_banner = Column(Boolean, default=False)
@ -114,7 +114,7 @@ class VariantAttribute(Base):
id = Column(String(26), primary_key=True)
variant_id = Column(String(26), ForeignKey("product_variants.variant_id", ondelete="CASCADE"), nullable=False)
attribute_id = Column(String(26), ForeignKey("attribute_types.attribute_id", ondelete="RESTRICT"), nullable=False)
attribute_value = Column(String(255), nullable=False)
attribute_value = Column(String(2000), nullable=False)
variant = relationship("ProductVariant", back_populates="attributes")
attribute_type = relationship("AttributeType")
@ -124,7 +124,7 @@ class VariantImage(Base):
image_id = Column(String(26), primary_key=True)
variant_id = Column(String(26), ForeignKey("product_variants.variant_id", ondelete="CASCADE"), nullable=False)
image_url = Column(String(500), nullable=False)
image_url = Column(String(2000), nullable=False)
sort_order = Column(Integer, default=0)
is_primary = Column(Boolean, default=False)

View file

@ -420,7 +420,7 @@ class AttributeTypeResponse(BaseModel):
# --- Variant Attribute ---
class VariantAttributeCreate(BaseModel):
attribute_id: str
attribute_value: str = Field(..., min_length=1, max_length=255)
attribute_value: str = Field(..., min_length=1, max_length=2000)
class VariantAttributeResponse(BaseModel):
id: str

View file

@ -784,42 +784,84 @@ class MigrationWorker:
color = variant_colors.get(v.variant_id, '')
brand_name = brands.get(p.brand_id, '')
# Priority 1: Direct media keys from CSV attributes
# Fetch raw media keys for this variant upfront (used by all priority levels)
raw_mk = variant_media_keys.get(v.variant_id)
raw_pmk = variant_parent_media_keys.get(v.variant_id)
c_mk = MediaResolver.canonicalize_media_key(raw_mk) if raw_mk else None
c_pmk = MediaResolver.canonicalize_media_key(raw_pmk) if raw_pmk else None
# Priority 2: Synthetic name/slug keys
short_slug = p.slug.split('-for-')[0] if '-for-' in p.slug else p.slug
key3 = MediaResolver.canonicalize_media_key(f'{short_slug} {color}')
key1 = MediaResolver.canonicalize_media_key(f'{brand_name} {p.name} {color}')
key2 = MediaResolver.canonicalize_media_key(f'{p.slug} {color}')
# Priority 0: Comma-separated media keys (multi-image sequence)
matched_mg_assets = []
seen_linked_asset_ids = set()
mk_source = raw_mk or raw_pmk
if mk_source and "," in mk_source:
sub_keys = [k.strip() for k in mk_source.split(",") if k.strip()]
for sk in sub_keys:
c_sk = MediaResolver.canonicalize_media_key(sk)
if c_sk and c_sk in mg_map:
for asset in mg_map[c_sk].media_assets:
if asset.id not in seen_linked_asset_ids:
seen_linked_asset_ids.add(asset.id)
matched_mg_assets.append(asset)
mg = None
for candidate in [c_mk, c_pmk, raw_mk, raw_pmk, key3, key1, key2]:
if candidate and candidate in mg_map:
mg = mg_map[candidate]
break
if not matched_mg_assets:
# Priority 1: Direct media keys from CSV attributes (already fetched above)
# Priority 3: Fuzzy token-similarity matching if exact key has minor variance
if not mg and (c_mk or c_pmk or p.name):
target_str = c_mk or c_pmk or f"{p.name} {color}"
target_tokens = set(re.findall(r'[a-zA-Z0-9]+', target_str.lower()))
if target_tokens:
best_cand, best_score = None, 0.0
for cand_key in mg_map.keys():
cand_tokens = set(re.findall(r'[a-zA-Z0-9]+', cand_key.lower()))
if not cand_tokens: continue
overlap = len(target_tokens & cand_tokens)
score = overlap / max(len(target_tokens), len(cand_tokens))
if score > best_score:
best_score = score
best_cand = cand_key
if best_score >= 0.55: # High confidence token overlap
mg = mg_map[best_cand]
# Priority 2: Synthetic name/slug keys
short_slug = p.slug.split('-for-')[0] if '-for-' in p.slug else p.slug
key3 = MediaResolver.canonicalize_media_key(f'{short_slug} {color}')
key1 = MediaResolver.canonicalize_media_key(f'{brand_name} {p.name} {color}')
key2 = MediaResolver.canonicalize_media_key(f'{p.slug} {color}')
if mg and mg.media_assets:
for candidate in [c_mk, c_pmk, raw_mk, raw_pmk, key3, key1, key2]:
if candidate and candidate in mg_map:
mg = mg_map[candidate]
break
# Priority 3: Fuzzy token-similarity matching if exact key has minor variance
if not mg and (c_mk or c_pmk or p.name):
target_str = c_mk or c_pmk or f"{p.name} {color}"
target_tokens = set(re.findall(r'[a-zA-Z0-9]+', target_str.lower()))
if target_tokens:
best_cand, best_score = None, 0.0
for cand_key in mg_map.keys():
cand_tokens = set(re.findall(r'[a-zA-Z0-9]+', cand_key.lower()))
if not cand_tokens: continue
overlap = len(target_tokens & cand_tokens)
score = overlap / max(len(target_tokens), len(cand_tokens))
if score > best_score:
best_score = score
best_cand = cand_key
if best_score >= 0.55: # High confidence token overlap
mg = mg_map[best_cand]
if matched_mg_assets:
if v.variant_id not in existing_vi_variant_ids:
for idx, asset in enumerate(matched_mg_assets):
vi = VariantImage(
image_id=str(ulid.ULID()),
variant_id=v.variant_id,
image_url=asset.cdn_url,
sort_order=idx,
is_primary=(idx == 0)
)
db.add(vi)
images_linked_count += 1
existing_vi_variant_ids.add(v.variant_id)
if p.product_id not in existing_pi_product_ids:
pi = ProductImage(
image_id=str(ulid.ULID()),
product_id=p.product_id,
image_url=matched_mg_assets[0].cdn_url,
alt_text=matched_mg_assets[0].original_filename or p.name,
sort_order=0,
is_banner=False
)
db.add(pi)
existing_pi_product_ids.add(p.product_id)
elif mg and mg.media_assets:
if v.variant_id not in existing_vi_variant_ids:
for idx, asset in enumerate(mg.media_assets):
vi = VariantImage(

View file

@ -76,6 +76,8 @@ class UpsertExecutor:
continue
sp = None
new_series_keys = []
new_model_keys = []
try:
sp = db_session.begin_nested()
# 0. Get device type
@ -249,6 +251,8 @@ class UpsertExecutor:
db_session.add(series_obj)
db_session.flush()
if series_cache is not None:
if s_key not in series_cache:
new_series_keys.append(s_key)
series_cache[s_key] = series_obj
if series_obj and device_type_val and not series_obj.device_type:
@ -285,6 +289,8 @@ class UpsertExecutor:
db_session.add(model_obj)
db_session.flush()
if model_cache is not None:
if m_key not in model_cache:
new_model_keys.append(m_key)
model_cache[m_key] = model_obj
if model_obj and device_type_val and not model_obj.device_type:
@ -546,38 +552,44 @@ class UpsertExecutor:
db_session.add(var_attr)
# CRITICAL: Always save media_key and parent_media_key separately.
# They are excluded from clean_attributes (to keep variant specs display clean)
# but MUST exist in variant_attributes for MEDIA_LINK to match and link images.
# Supports comma-separated keys as well as single keys.
for mk_code, mk_name in (("media_key", "media_key"), ("parent_media_key", "parent_media_key")):
raw_mk_val = all_attributes.get(mk_code)
if raw_mk_val:
canonical_val = MediaResolver.canonicalize_media_key(str(raw_mk_val)) or str(raw_mk_val)
if canonical_val:
mk_attr_type = None
if attr_type_cache is not None and mk_code in attr_type_cache:
mk_attr_type = attr_type_cache[mk_code]
else:
mk_attr_type = db_session.query(AttributeType).filter(AttributeType.code == mk_code).first()
if not mk_attr_type:
mk_at_id = str(ulid.ULID())
mk_attr_type = AttributeType(
attribute_id=mk_at_id,
name=mk_name,
code=mk_code,
status="active"
)
db_session.add(mk_attr_type)
if attr_type_cache is not None:
attr_type_cache[mk_code] = mk_attr_type
# Split on commas for multi-image references
sub_keys = [k.strip() for k in str(raw_mk_val).split(",") if k.strip()]
if not sub_keys and str(raw_mk_val).strip():
sub_keys = [str(raw_mk_val).strip()]
db_session.add(VariantAttribute(
id=str(ulid.ULID()),
variant_id=variant.variant_id,
attribute_id=mk_attr_type.attribute_id,
attribute_value=canonical_val
))
mk_attr_type = None
if attr_type_cache is not None and mk_code in attr_type_cache:
mk_attr_type = attr_type_cache[mk_code]
else:
mk_attr_type = db_session.query(AttributeType).filter(AttributeType.code == mk_code).first()
if not mk_attr_type:
mk_at_id = str(ulid.ULID())
mk_attr_type = AttributeType(
attribute_id=mk_at_id,
name=mk_name,
code=mk_code,
status="active"
)
db_session.add(mk_attr_type)
if attr_type_cache is not None:
attr_type_cache[mk_code] = mk_attr_type
# 6. Attach Variant Images via Smart 3-Tier Canonical Media Key Resolver
for sk in sub_keys:
canonical_val = MediaResolver.canonicalize_media_key(sk) or sk
if canonical_val:
canonical_val = canonical_val[:2000]
db_session.add(VariantAttribute(
id=str(ulid.ULID()),
variant_id=variant.variant_id,
attribute_id=mk_attr_type.attribute_id,
attribute_value=canonical_val
))
# 6. Attach Variant Images via Smart Canonical Media Key Resolver (Supporting Comma-Separated Keys)
raw_media_key = mapped_row.get("media_key") or mapped_row.get("parent_media_key")
# Tier 2 Fallback: Match by SKU if no media_key column is in Excel
@ -588,24 +600,44 @@ class UpsertExecutor:
if not raw_media_key:
raw_media_key = mapped_row.get("name") or mapped_row.get("parent_name")
media_key = MediaResolver.canonicalize_media_key(raw_media_key)
if raw_media_key:
mk_tokens = [k.strip() for k in str(raw_media_key).split(",") if k.strip()]
if not mk_tokens and str(raw_media_key).strip():
mk_tokens = [str(raw_media_key).strip()]
if media_key:
m_group = None
if media_group_cache is not None and media_key in media_group_cache:
m_group = media_group_cache[media_key]
else:
m_group = db_session.query(MediaGroup).filter(MediaGroup.media_key == media_key).first()
if not m_group and mapped_row.get("parent_media_key"):
p_key = MediaResolver.canonicalize_media_key(mapped_row.get("parent_media_key"))
if p_key:
m_group = db_session.query(MediaGroup).filter(MediaGroup.media_key == p_key).first()
if media_group_cache is not None:
media_group_cache[media_key] = m_group
all_matched_assets = []
seen_asset_ids = set()
if m_group and m_group.media_assets:
for token_key in mk_tokens:
c_media_key = MediaResolver.canonicalize_media_key(token_key)
if not c_media_key:
continue
m_group = None
if media_group_cache is not None and c_media_key in media_group_cache:
m_group = media_group_cache[c_media_key]
else:
m_group = db_session.query(MediaGroup).filter(MediaGroup.media_key == c_media_key).first()
if not m_group and mapped_row.get("parent_media_key"):
p_tokens = [pk.strip() for pk in str(mapped_row.get("parent_media_key")).split(",") if pk.strip()]
for p_tok in p_tokens:
p_key = MediaResolver.canonicalize_media_key(p_tok)
if p_key:
m_group = db_session.query(MediaGroup).filter(MediaGroup.media_key == p_key).first()
if m_group:
break
if media_group_cache is not None:
media_group_cache[c_media_key] = m_group
if m_group and m_group.media_assets:
for asset in m_group.media_assets:
if asset.id not in seen_asset_ids:
seen_asset_ids.add(asset.id)
all_matched_assets.append(asset)
if all_matched_assets:
db_session.query(VariantImage).filter(VariantImage.variant_id == variant.variant_id).delete()
for idx, asset in enumerate(m_group.media_assets):
for idx, asset in enumerate(all_matched_assets):
vi_id = str(ulid.ULID())
var_img = VariantImage(
image_id=vi_id,
@ -770,6 +802,13 @@ class UpsertExecutor:
sp.rollback()
except Exception:
pass
# Evict uncommitted series and models from cache to prevent cascading foreign key failures
if series_cache is not None:
for sk in new_series_keys:
series_cache.pop(sk, None)
if model_cache is not None:
for mk in new_model_keys:
model_cache.pop(mk, None)
fail_count += 1
runtime_errors.append({
"row_number": row_number,