fix: resolve MEDIA_LINK NameError crash and expand VARCHAR columns to 2000
- Fix NameError in link_media_groups_to_catalog: raw_mk/raw_pmk were used before assignment in Priority 0 comma-split block; moved fetches to top of variant loop so all priority levels share the same resolved values - Expand variant_attributes.attribute_value to String(2000) for comma-separated multi-image media keys - Expand products, device_models, device_series, media_library, migration_media_items URL/path/name columns to String(2000) - Update Pydantic VariantAttributeCreate.attribute_value max_length=2000 - Add rollback cache eviction for new series/model keys (fixes MySQL 1452) - Add comma-separated media key splitting in upsert_executor for multi-image variant image attachment - Add Priority 0 comma-separated MEDIA_LINK support in migration_worker
This commit is contained in:
parent
7c7c474e47
commit
5dc56b2687
6 changed files with 169 additions and 88 deletions
|
|
@ -10,7 +10,7 @@ class DeviceSeries(Base):
|
|||
|
||||
series_id = Column(String(26), primary_key=True)
|
||||
brand_id = Column(String(26), ForeignKey("brands.brand_id", ondelete="SET NULL"), nullable=True)
|
||||
name = Column(String(100), nullable=False)
|
||||
name = Column(String(2000), nullable=False)
|
||||
slug = Column(String(100), unique=True, nullable=False)
|
||||
device_type = Column(String(50), nullable=True) # laptop, tablet, mobile
|
||||
sort_order = Column(Integer, default=0)
|
||||
|
|
@ -25,10 +25,10 @@ class DeviceModel(Base):
|
|||
model_id = Column(String(26), primary_key=True)
|
||||
series_id = Column(String(26), ForeignKey("device_series.series_id", ondelete="SET NULL"), nullable=True)
|
||||
brand_id = Column(String(26), ForeignKey("brands.brand_id", ondelete="SET NULL"), nullable=True)
|
||||
name = Column(String(100), nullable=False)
|
||||
name = Column(String(2000), nullable=False)
|
||||
slug = Column(String(100), unique=True, nullable=False)
|
||||
device_type = Column(String(50), nullable=True) # laptop, tablet, mobile
|
||||
full_path = Column(String(500), nullable=False)
|
||||
full_path = Column(String(2000), nullable=False)
|
||||
release_year = Column(Integer, nullable=True)
|
||||
image_url = Column(String(500), nullable=True)
|
||||
is_active = Column(Boolean, default=True)
|
||||
|
|
|
|||
|
|
@ -119,16 +119,16 @@ class MediaAsset(Base):
|
|||
|
||||
id = Column(String(36), primary_key=True, default=generate_uuid)
|
||||
media_group_id = Column(String(36), ForeignKey("media_groups.id", ondelete="CASCADE"), nullable=False)
|
||||
original_filename = Column(String(255), nullable=False)
|
||||
stored_filename = Column(String(255), nullable=False)
|
||||
original_filename = Column(String(2000), nullable=False)
|
||||
stored_filename = Column(String(2000), nullable=False)
|
||||
mime_type = Column(String(50), nullable=False)
|
||||
file_size_bytes = Column(BigInteger, nullable=False)
|
||||
width = Column(Integer, nullable=False, default=0)
|
||||
height = Column(Integer, nullable=False, default=0)
|
||||
sha256_checksum = Column(String(64), unique=True, nullable=False, index=True) # Global unique SHA-256 asset
|
||||
cdn_url = Column(String(512), nullable=False)
|
||||
thumbnail_url = Column(String(512), nullable=False)
|
||||
storage_path = Column(String(512), nullable=True)
|
||||
cdn_url = Column(String(2000), nullable=False)
|
||||
thumbnail_url = Column(String(2000), nullable=False)
|
||||
storage_path = Column(String(2000), nullable=True)
|
||||
exif_metadata = Column(JSON, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
|
|
@ -207,12 +207,12 @@ class MigrationMediaItem(Base):
|
|||
id = Column(String(36), primary_key=True, default=generate_uuid)
|
||||
job_id = Column(String(36), ForeignKey("migration_jobs.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
batch_number = Column(Integer, nullable=False, default=0)
|
||||
file_name = Column(String(255), nullable=False)
|
||||
file_name = Column(String(2000), nullable=False)
|
||||
archive_name = Column(String(255), nullable=True)
|
||||
zip_entry_path = Column(String(512), nullable=True)
|
||||
zip_entry_path = Column(String(2000), nullable=True)
|
||||
media_key = Column(String(255), nullable=False, index=True)
|
||||
sha256 = Column(String(64), nullable=True, index=True) # NOT unique (allows multiple source references to map to 1 MediaAsset)
|
||||
storage_path = Column(String(512), nullable=True)
|
||||
storage_path = Column(String(2000), nullable=True)
|
||||
status = Column(SQLEnum(MediaItemStatusEnum), nullable=False, default=MediaItemStatusEnum.PENDING, index=True)
|
||||
error = Column(Text, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
|
|
|||
|
|
@ -25,14 +25,14 @@ class Product(Base):
|
|||
device_series_id = Column(String(26), ForeignKey("device_series.series_id", ondelete="SET NULL"), nullable=True)
|
||||
device_model_id = Column(String(26), ForeignKey("device_models.model_id", ondelete="SET NULL"), nullable=True)
|
||||
device_type = Column(String(50), nullable=True) # laptop, tablet, mobile
|
||||
name = Column(String(255), nullable=False)
|
||||
name = Column(String(2000), nullable=False)
|
||||
name_normalized = Column(String(255), nullable=True, index=True)
|
||||
model_sentinel = Column(String(36), nullable=False, default="GENERIC")
|
||||
slug = Column(String(255), unique=True, nullable=False)
|
||||
full_path = Column(String(500), nullable=False)
|
||||
full_path = Column(String(2000), nullable=False)
|
||||
description = Column(Text, nullable=True)
|
||||
warranty_type = Column(String(100), nullable=True)
|
||||
warranty_summary = Column(String(500), nullable=True)
|
||||
warranty_summary = Column(String(2000), nullable=True)
|
||||
seo_title = Column(Text, nullable=True)
|
||||
seo_description = Column(Text, nullable=True)
|
||||
meta_keywords = Column(Text, nullable=True)
|
||||
|
|
@ -82,8 +82,8 @@ class ProductImage(Base):
|
|||
|
||||
image_id = Column(String(26), primary_key=True)
|
||||
product_id = Column(String(26), ForeignKey("products.product_id", ondelete="CASCADE"), nullable=False)
|
||||
image_url = Column(String(500), nullable=False)
|
||||
alt_text = Column(String(255), nullable=True)
|
||||
image_url = Column(String(2000), nullable=False)
|
||||
alt_text = Column(String(2000), nullable=True)
|
||||
sort_order = Column(Integer, default=0)
|
||||
is_banner = Column(Boolean, default=False)
|
||||
|
||||
|
|
@ -114,7 +114,7 @@ class VariantAttribute(Base):
|
|||
id = Column(String(26), primary_key=True)
|
||||
variant_id = Column(String(26), ForeignKey("product_variants.variant_id", ondelete="CASCADE"), nullable=False)
|
||||
attribute_id = Column(String(26), ForeignKey("attribute_types.attribute_id", ondelete="RESTRICT"), nullable=False)
|
||||
attribute_value = Column(String(255), nullable=False)
|
||||
attribute_value = Column(String(2000), nullable=False)
|
||||
|
||||
variant = relationship("ProductVariant", back_populates="attributes")
|
||||
attribute_type = relationship("AttributeType")
|
||||
|
|
@ -124,7 +124,7 @@ class VariantImage(Base):
|
|||
|
||||
image_id = Column(String(26), primary_key=True)
|
||||
variant_id = Column(String(26), ForeignKey("product_variants.variant_id", ondelete="CASCADE"), nullable=False)
|
||||
image_url = Column(String(500), nullable=False)
|
||||
image_url = Column(String(2000), nullable=False)
|
||||
sort_order = Column(Integer, default=0)
|
||||
is_primary = Column(Boolean, default=False)
|
||||
|
||||
|
|
|
|||
|
|
@ -420,7 +420,7 @@ class AttributeTypeResponse(BaseModel):
|
|||
# --- Variant Attribute ---
|
||||
class VariantAttributeCreate(BaseModel):
|
||||
attribute_id: str
|
||||
attribute_value: str = Field(..., min_length=1, max_length=255)
|
||||
attribute_value: str = Field(..., min_length=1, max_length=2000)
|
||||
|
||||
class VariantAttributeResponse(BaseModel):
|
||||
id: str
|
||||
|
|
|
|||
|
|
@ -784,19 +784,36 @@ class MigrationWorker:
|
|||
color = variant_colors.get(v.variant_id, '')
|
||||
brand_name = brands.get(p.brand_id, '')
|
||||
|
||||
# Priority 1: Direct media keys from CSV attributes
|
||||
# Fetch raw media keys for this variant upfront (used by all priority levels)
|
||||
raw_mk = variant_media_keys.get(v.variant_id)
|
||||
raw_pmk = variant_parent_media_keys.get(v.variant_id)
|
||||
c_mk = MediaResolver.canonicalize_media_key(raw_mk) if raw_mk else None
|
||||
c_pmk = MediaResolver.canonicalize_media_key(raw_pmk) if raw_pmk else None
|
||||
|
||||
# Priority 0: Comma-separated media keys (multi-image sequence)
|
||||
matched_mg_assets = []
|
||||
seen_linked_asset_ids = set()
|
||||
mk_source = raw_mk or raw_pmk
|
||||
if mk_source and "," in mk_source:
|
||||
sub_keys = [k.strip() for k in mk_source.split(",") if k.strip()]
|
||||
for sk in sub_keys:
|
||||
c_sk = MediaResolver.canonicalize_media_key(sk)
|
||||
if c_sk and c_sk in mg_map:
|
||||
for asset in mg_map[c_sk].media_assets:
|
||||
if asset.id not in seen_linked_asset_ids:
|
||||
seen_linked_asset_ids.add(asset.id)
|
||||
matched_mg_assets.append(asset)
|
||||
|
||||
mg = None
|
||||
if not matched_mg_assets:
|
||||
# Priority 1: Direct media keys from CSV attributes (already fetched above)
|
||||
|
||||
# Priority 2: Synthetic name/slug keys
|
||||
short_slug = p.slug.split('-for-')[0] if '-for-' in p.slug else p.slug
|
||||
key3 = MediaResolver.canonicalize_media_key(f'{short_slug} {color}')
|
||||
key1 = MediaResolver.canonicalize_media_key(f'{brand_name} {p.name} {color}')
|
||||
key2 = MediaResolver.canonicalize_media_key(f'{p.slug} {color}')
|
||||
|
||||
mg = None
|
||||
for candidate in [c_mk, c_pmk, raw_mk, raw_pmk, key3, key1, key2]:
|
||||
if candidate and candidate in mg_map:
|
||||
mg = mg_map[candidate]
|
||||
|
|
@ -819,7 +836,32 @@ class MigrationWorker:
|
|||
if best_score >= 0.55: # High confidence token overlap
|
||||
mg = mg_map[best_cand]
|
||||
|
||||
if mg and mg.media_assets:
|
||||
if matched_mg_assets:
|
||||
if v.variant_id not in existing_vi_variant_ids:
|
||||
for idx, asset in enumerate(matched_mg_assets):
|
||||
vi = VariantImage(
|
||||
image_id=str(ulid.ULID()),
|
||||
variant_id=v.variant_id,
|
||||
image_url=asset.cdn_url,
|
||||
sort_order=idx,
|
||||
is_primary=(idx == 0)
|
||||
)
|
||||
db.add(vi)
|
||||
images_linked_count += 1
|
||||
existing_vi_variant_ids.add(v.variant_id)
|
||||
|
||||
if p.product_id not in existing_pi_product_ids:
|
||||
pi = ProductImage(
|
||||
image_id=str(ulid.ULID()),
|
||||
product_id=p.product_id,
|
||||
image_url=matched_mg_assets[0].cdn_url,
|
||||
alt_text=matched_mg_assets[0].original_filename or p.name,
|
||||
sort_order=0,
|
||||
is_banner=False
|
||||
)
|
||||
db.add(pi)
|
||||
existing_pi_product_ids.add(p.product_id)
|
||||
elif mg and mg.media_assets:
|
||||
if v.variant_id not in existing_vi_variant_ids:
|
||||
for idx, asset in enumerate(mg.media_assets):
|
||||
vi = VariantImage(
|
||||
|
|
|
|||
|
|
@ -76,6 +76,8 @@ class UpsertExecutor:
|
|||
continue
|
||||
|
||||
sp = None
|
||||
new_series_keys = []
|
||||
new_model_keys = []
|
||||
try:
|
||||
sp = db_session.begin_nested()
|
||||
# 0. Get device type
|
||||
|
|
@ -249,6 +251,8 @@ class UpsertExecutor:
|
|||
db_session.add(series_obj)
|
||||
db_session.flush()
|
||||
if series_cache is not None:
|
||||
if s_key not in series_cache:
|
||||
new_series_keys.append(s_key)
|
||||
series_cache[s_key] = series_obj
|
||||
|
||||
if series_obj and device_type_val and not series_obj.device_type:
|
||||
|
|
@ -285,6 +289,8 @@ class UpsertExecutor:
|
|||
db_session.add(model_obj)
|
||||
db_session.flush()
|
||||
if model_cache is not None:
|
||||
if m_key not in model_cache:
|
||||
new_model_keys.append(m_key)
|
||||
model_cache[m_key] = model_obj
|
||||
|
||||
if model_obj and device_type_val and not model_obj.device_type:
|
||||
|
|
@ -546,13 +552,15 @@ class UpsertExecutor:
|
|||
db_session.add(var_attr)
|
||||
|
||||
# CRITICAL: Always save media_key and parent_media_key separately.
|
||||
# They are excluded from clean_attributes (to keep variant specs display clean)
|
||||
# but MUST exist in variant_attributes for MEDIA_LINK to match and link images.
|
||||
# Supports comma-separated keys as well as single keys.
|
||||
for mk_code, mk_name in (("media_key", "media_key"), ("parent_media_key", "parent_media_key")):
|
||||
raw_mk_val = all_attributes.get(mk_code)
|
||||
if raw_mk_val:
|
||||
canonical_val = MediaResolver.canonicalize_media_key(str(raw_mk_val)) or str(raw_mk_val)
|
||||
if canonical_val:
|
||||
# Split on commas for multi-image references
|
||||
sub_keys = [k.strip() for k in str(raw_mk_val).split(",") if k.strip()]
|
||||
if not sub_keys and str(raw_mk_val).strip():
|
||||
sub_keys = [str(raw_mk_val).strip()]
|
||||
|
||||
mk_attr_type = None
|
||||
if attr_type_cache is not None and mk_code in attr_type_cache:
|
||||
mk_attr_type = attr_type_cache[mk_code]
|
||||
|
|
@ -570,6 +578,10 @@ class UpsertExecutor:
|
|||
if attr_type_cache is not None:
|
||||
attr_type_cache[mk_code] = mk_attr_type
|
||||
|
||||
for sk in sub_keys:
|
||||
canonical_val = MediaResolver.canonicalize_media_key(sk) or sk
|
||||
if canonical_val:
|
||||
canonical_val = canonical_val[:2000]
|
||||
db_session.add(VariantAttribute(
|
||||
id=str(ulid.ULID()),
|
||||
variant_id=variant.variant_id,
|
||||
|
|
@ -577,7 +589,7 @@ class UpsertExecutor:
|
|||
attribute_value=canonical_val
|
||||
))
|
||||
|
||||
# 6. Attach Variant Images via Smart 3-Tier Canonical Media Key Resolver
|
||||
# 6. Attach Variant Images via Smart Canonical Media Key Resolver (Supporting Comma-Separated Keys)
|
||||
raw_media_key = mapped_row.get("media_key") or mapped_row.get("parent_media_key")
|
||||
|
||||
# Tier 2 Fallback: Match by SKU if no media_key column is in Excel
|
||||
|
|
@ -588,24 +600,44 @@ class UpsertExecutor:
|
|||
if not raw_media_key:
|
||||
raw_media_key = mapped_row.get("name") or mapped_row.get("parent_name")
|
||||
|
||||
media_key = MediaResolver.canonicalize_media_key(raw_media_key)
|
||||
if raw_media_key:
|
||||
mk_tokens = [k.strip() for k in str(raw_media_key).split(",") if k.strip()]
|
||||
if not mk_tokens and str(raw_media_key).strip():
|
||||
mk_tokens = [str(raw_media_key).strip()]
|
||||
|
||||
all_matched_assets = []
|
||||
seen_asset_ids = set()
|
||||
|
||||
for token_key in mk_tokens:
|
||||
c_media_key = MediaResolver.canonicalize_media_key(token_key)
|
||||
if not c_media_key:
|
||||
continue
|
||||
|
||||
if media_key:
|
||||
m_group = None
|
||||
if media_group_cache is not None and media_key in media_group_cache:
|
||||
m_group = media_group_cache[media_key]
|
||||
if media_group_cache is not None and c_media_key in media_group_cache:
|
||||
m_group = media_group_cache[c_media_key]
|
||||
else:
|
||||
m_group = db_session.query(MediaGroup).filter(MediaGroup.media_key == media_key).first()
|
||||
m_group = db_session.query(MediaGroup).filter(MediaGroup.media_key == c_media_key).first()
|
||||
if not m_group and mapped_row.get("parent_media_key"):
|
||||
p_key = MediaResolver.canonicalize_media_key(mapped_row.get("parent_media_key"))
|
||||
p_tokens = [pk.strip() for pk in str(mapped_row.get("parent_media_key")).split(",") if pk.strip()]
|
||||
for p_tok in p_tokens:
|
||||
p_key = MediaResolver.canonicalize_media_key(p_tok)
|
||||
if p_key:
|
||||
m_group = db_session.query(MediaGroup).filter(MediaGroup.media_key == p_key).first()
|
||||
if m_group:
|
||||
break
|
||||
if media_group_cache is not None:
|
||||
media_group_cache[media_key] = m_group
|
||||
media_group_cache[c_media_key] = m_group
|
||||
|
||||
if m_group and m_group.media_assets:
|
||||
for asset in m_group.media_assets:
|
||||
if asset.id not in seen_asset_ids:
|
||||
seen_asset_ids.add(asset.id)
|
||||
all_matched_assets.append(asset)
|
||||
|
||||
if all_matched_assets:
|
||||
db_session.query(VariantImage).filter(VariantImage.variant_id == variant.variant_id).delete()
|
||||
for idx, asset in enumerate(m_group.media_assets):
|
||||
for idx, asset in enumerate(all_matched_assets):
|
||||
vi_id = str(ulid.ULID())
|
||||
var_img = VariantImage(
|
||||
image_id=vi_id,
|
||||
|
|
@ -770,6 +802,13 @@ class UpsertExecutor:
|
|||
sp.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
# Evict uncommitted series and models from cache to prevent cascading foreign key failures
|
||||
if series_cache is not None:
|
||||
for sk in new_series_keys:
|
||||
series_cache.pop(sk, None)
|
||||
if model_cache is not None:
|
||||
for mk in new_model_keys:
|
||||
model_cache.pop(mk, None)
|
||||
fail_count += 1
|
||||
runtime_errors.append({
|
||||
"row_number": row_number,
|
||||
|
|
|
|||
Loading…
Reference in a new issue