diff --git a/app/models/DeviceCatalogModel.py b/app/models/DeviceCatalogModel.py index dbcfb6f..e1dc015 100644 --- a/app/models/DeviceCatalogModel.py +++ b/app/models/DeviceCatalogModel.py @@ -10,7 +10,7 @@ class DeviceSeries(Base): series_id = Column(String(26), primary_key=True) brand_id = Column(String(26), ForeignKey("brands.brand_id", ondelete="SET NULL"), nullable=True) - name = Column(String(100), nullable=False) + name = Column(String(2000), nullable=False) slug = Column(String(100), unique=True, nullable=False) device_type = Column(String(50), nullable=True) # laptop, tablet, mobile sort_order = Column(Integer, default=0) @@ -25,10 +25,10 @@ class DeviceModel(Base): model_id = Column(String(26), primary_key=True) series_id = Column(String(26), ForeignKey("device_series.series_id", ondelete="SET NULL"), nullable=True) brand_id = Column(String(26), ForeignKey("brands.brand_id", ondelete="SET NULL"), nullable=True) - name = Column(String(100), nullable=False) + name = Column(String(2000), nullable=False) slug = Column(String(100), unique=True, nullable=False) device_type = Column(String(50), nullable=True) # laptop, tablet, mobile - full_path = Column(String(500), nullable=False) + full_path = Column(String(2000), nullable=False) release_year = Column(Integer, nullable=True) image_url = Column(String(500), nullable=True) is_active = Column(Boolean, default=True) diff --git a/app/models/MigrationModel.py b/app/models/MigrationModel.py index 0fd0e17..259bc89 100644 --- a/app/models/MigrationModel.py +++ b/app/models/MigrationModel.py @@ -119,16 +119,16 @@ class MediaAsset(Base): id = Column(String(36), primary_key=True, default=generate_uuid) media_group_id = Column(String(36), ForeignKey("media_groups.id", ondelete="CASCADE"), nullable=False) - original_filename = Column(String(255), nullable=False) - stored_filename = Column(String(255), nullable=False) + original_filename = Column(String(2000), nullable=False) + stored_filename = Column(String(2000), nullable=False) mime_type = Column(String(50), nullable=False) file_size_bytes = Column(BigInteger, nullable=False) width = Column(Integer, nullable=False, default=0) height = Column(Integer, nullable=False, default=0) sha256_checksum = Column(String(64), unique=True, nullable=False, index=True) # Global unique SHA-256 asset - cdn_url = Column(String(512), nullable=False) - thumbnail_url = Column(String(512), nullable=False) - storage_path = Column(String(512), nullable=True) + cdn_url = Column(String(2000), nullable=False) + thumbnail_url = Column(String(2000), nullable=False) + storage_path = Column(String(2000), nullable=True) exif_metadata = Column(JSON, nullable=True) created_at = Column(DateTime, default=datetime.utcnow) @@ -207,12 +207,12 @@ class MigrationMediaItem(Base): id = Column(String(36), primary_key=True, default=generate_uuid) job_id = Column(String(36), ForeignKey("migration_jobs.id", ondelete="CASCADE"), nullable=False, index=True) batch_number = Column(Integer, nullable=False, default=0) - file_name = Column(String(255), nullable=False) + file_name = Column(String(2000), nullable=False) archive_name = Column(String(255), nullable=True) - zip_entry_path = Column(String(512), nullable=True) + zip_entry_path = Column(String(2000), nullable=True) media_key = Column(String(255), nullable=False, index=True) sha256 = Column(String(64), nullable=True, index=True) # NOT unique (allows multiple source references to map to 1 MediaAsset) - storage_path = Column(String(512), nullable=True) + storage_path = Column(String(2000), nullable=True) status = Column(SQLEnum(MediaItemStatusEnum), nullable=False, default=MediaItemStatusEnum.PENDING, index=True) error = Column(Text, nullable=True) created_at = Column(DateTime, default=datetime.utcnow) diff --git a/app/models/ProductModel.py b/app/models/ProductModel.py index f0ae339..5092207 100644 --- a/app/models/ProductModel.py +++ b/app/models/ProductModel.py @@ -25,14 +25,14 @@ class Product(Base): device_series_id = Column(String(26), ForeignKey("device_series.series_id", ondelete="SET NULL"), nullable=True) device_model_id = Column(String(26), ForeignKey("device_models.model_id", ondelete="SET NULL"), nullable=True) device_type = Column(String(50), nullable=True) # laptop, tablet, mobile - name = Column(String(255), nullable=False) + name = Column(String(2000), nullable=False) name_normalized = Column(String(255), nullable=True, index=True) model_sentinel = Column(String(36), nullable=False, default="GENERIC") slug = Column(String(255), unique=True, nullable=False) - full_path = Column(String(500), nullable=False) + full_path = Column(String(2000), nullable=False) description = Column(Text, nullable=True) warranty_type = Column(String(100), nullable=True) - warranty_summary = Column(String(500), nullable=True) + warranty_summary = Column(String(2000), nullable=True) seo_title = Column(Text, nullable=True) seo_description = Column(Text, nullable=True) meta_keywords = Column(Text, nullable=True) @@ -82,8 +82,8 @@ class ProductImage(Base): image_id = Column(String(26), primary_key=True) product_id = Column(String(26), ForeignKey("products.product_id", ondelete="CASCADE"), nullable=False) - image_url = Column(String(500), nullable=False) - alt_text = Column(String(255), nullable=True) + image_url = Column(String(2000), nullable=False) + alt_text = Column(String(2000), nullable=True) sort_order = Column(Integer, default=0) is_banner = Column(Boolean, default=False) @@ -114,7 +114,7 @@ class VariantAttribute(Base): id = Column(String(26), primary_key=True) variant_id = Column(String(26), ForeignKey("product_variants.variant_id", ondelete="CASCADE"), nullable=False) attribute_id = Column(String(26), ForeignKey("attribute_types.attribute_id", ondelete="RESTRICT"), nullable=False) - attribute_value = Column(String(255), nullable=False) + attribute_value = Column(String(2000), nullable=False) variant = relationship("ProductVariant", back_populates="attributes") attribute_type = relationship("AttributeType") @@ -124,7 +124,7 @@ class VariantImage(Base): image_id = Column(String(26), primary_key=True) variant_id = Column(String(26), ForeignKey("product_variants.variant_id", ondelete="CASCADE"), nullable=False) - image_url = Column(String(500), nullable=False) + image_url = Column(String(2000), nullable=False) sort_order = Column(Integer, default=0) is_primary = Column(Boolean, default=False) diff --git a/app/schemas/Catalog.py b/app/schemas/Catalog.py index c3db0d5..2e5618c 100644 --- a/app/schemas/Catalog.py +++ b/app/schemas/Catalog.py @@ -420,7 +420,7 @@ class AttributeTypeResponse(BaseModel): # --- Variant Attribute --- class VariantAttributeCreate(BaseModel): attribute_id: str - attribute_value: str = Field(..., min_length=1, max_length=255) + attribute_value: str = Field(..., min_length=1, max_length=2000) class VariantAttributeResponse(BaseModel): id: str diff --git a/app/services/migration_engine/migration_worker.py b/app/services/migration_engine/migration_worker.py index 2434f98..04a2ae3 100644 --- a/app/services/migration_engine/migration_worker.py +++ b/app/services/migration_engine/migration_worker.py @@ -784,42 +784,84 @@ class MigrationWorker: color = variant_colors.get(v.variant_id, '') brand_name = brands.get(p.brand_id, '') - # Priority 1: Direct media keys from CSV attributes + # Fetch raw media keys for this variant upfront (used by all priority levels) raw_mk = variant_media_keys.get(v.variant_id) raw_pmk = variant_parent_media_keys.get(v.variant_id) c_mk = MediaResolver.canonicalize_media_key(raw_mk) if raw_mk else None c_pmk = MediaResolver.canonicalize_media_key(raw_pmk) if raw_pmk else None - # Priority 2: Synthetic name/slug keys - short_slug = p.slug.split('-for-')[0] if '-for-' in p.slug else p.slug - key3 = MediaResolver.canonicalize_media_key(f'{short_slug} {color}') - key1 = MediaResolver.canonicalize_media_key(f'{brand_name} {p.name} {color}') - key2 = MediaResolver.canonicalize_media_key(f'{p.slug} {color}') + # Priority 0: Comma-separated media keys (multi-image sequence) + matched_mg_assets = [] + seen_linked_asset_ids = set() + mk_source = raw_mk or raw_pmk + if mk_source and "," in mk_source: + sub_keys = [k.strip() for k in mk_source.split(",") if k.strip()] + for sk in sub_keys: + c_sk = MediaResolver.canonicalize_media_key(sk) + if c_sk and c_sk in mg_map: + for asset in mg_map[c_sk].media_assets: + if asset.id not in seen_linked_asset_ids: + seen_linked_asset_ids.add(asset.id) + matched_mg_assets.append(asset) mg = None - for candidate in [c_mk, c_pmk, raw_mk, raw_pmk, key3, key1, key2]: - if candidate and candidate in mg_map: - mg = mg_map[candidate] - break + if not matched_mg_assets: + # Priority 1: Direct media keys from CSV attributes (already fetched above) - # Priority 3: Fuzzy token-similarity matching if exact key has minor variance - if not mg and (c_mk or c_pmk or p.name): - target_str = c_mk or c_pmk or f"{p.name} {color}" - target_tokens = set(re.findall(r'[a-zA-Z0-9]+', target_str.lower())) - if target_tokens: - best_cand, best_score = None, 0.0 - for cand_key in mg_map.keys(): - cand_tokens = set(re.findall(r'[a-zA-Z0-9]+', cand_key.lower())) - if not cand_tokens: continue - overlap = len(target_tokens & cand_tokens) - score = overlap / max(len(target_tokens), len(cand_tokens)) - if score > best_score: - best_score = score - best_cand = cand_key - if best_score >= 0.55: # High confidence token overlap - mg = mg_map[best_cand] + # Priority 2: Synthetic name/slug keys + short_slug = p.slug.split('-for-')[0] if '-for-' in p.slug else p.slug + key3 = MediaResolver.canonicalize_media_key(f'{short_slug} {color}') + key1 = MediaResolver.canonicalize_media_key(f'{brand_name} {p.name} {color}') + key2 = MediaResolver.canonicalize_media_key(f'{p.slug} {color}') - if mg and mg.media_assets: + for candidate in [c_mk, c_pmk, raw_mk, raw_pmk, key3, key1, key2]: + if candidate and candidate in mg_map: + mg = mg_map[candidate] + break + + # Priority 3: Fuzzy token-similarity matching if exact key has minor variance + if not mg and (c_mk or c_pmk or p.name): + target_str = c_mk or c_pmk or f"{p.name} {color}" + target_tokens = set(re.findall(r'[a-zA-Z0-9]+', target_str.lower())) + if target_tokens: + best_cand, best_score = None, 0.0 + for cand_key in mg_map.keys(): + cand_tokens = set(re.findall(r'[a-zA-Z0-9]+', cand_key.lower())) + if not cand_tokens: continue + overlap = len(target_tokens & cand_tokens) + score = overlap / max(len(target_tokens), len(cand_tokens)) + if score > best_score: + best_score = score + best_cand = cand_key + if best_score >= 0.55: # High confidence token overlap + mg = mg_map[best_cand] + + if matched_mg_assets: + if v.variant_id not in existing_vi_variant_ids: + for idx, asset in enumerate(matched_mg_assets): + vi = VariantImage( + image_id=str(ulid.ULID()), + variant_id=v.variant_id, + image_url=asset.cdn_url, + sort_order=idx, + is_primary=(idx == 0) + ) + db.add(vi) + images_linked_count += 1 + existing_vi_variant_ids.add(v.variant_id) + + if p.product_id not in existing_pi_product_ids: + pi = ProductImage( + image_id=str(ulid.ULID()), + product_id=p.product_id, + image_url=matched_mg_assets[0].cdn_url, + alt_text=matched_mg_assets[0].original_filename or p.name, + sort_order=0, + is_banner=False + ) + db.add(pi) + existing_pi_product_ids.add(p.product_id) + elif mg and mg.media_assets: if v.variant_id not in existing_vi_variant_ids: for idx, asset in enumerate(mg.media_assets): vi = VariantImage( diff --git a/app/services/migration_engine/upsert_executor.py b/app/services/migration_engine/upsert_executor.py index dc33a52..c7635b6 100644 --- a/app/services/migration_engine/upsert_executor.py +++ b/app/services/migration_engine/upsert_executor.py @@ -76,6 +76,8 @@ class UpsertExecutor: continue sp = None + new_series_keys = [] + new_model_keys = [] try: sp = db_session.begin_nested() # 0. Get device type @@ -249,6 +251,8 @@ class UpsertExecutor: db_session.add(series_obj) db_session.flush() if series_cache is not None: + if s_key not in series_cache: + new_series_keys.append(s_key) series_cache[s_key] = series_obj if series_obj and device_type_val and not series_obj.device_type: @@ -285,6 +289,8 @@ class UpsertExecutor: db_session.add(model_obj) db_session.flush() if model_cache is not None: + if m_key not in model_cache: + new_model_keys.append(m_key) model_cache[m_key] = model_obj if model_obj and device_type_val and not model_obj.device_type: @@ -546,38 +552,44 @@ class UpsertExecutor: db_session.add(var_attr) # CRITICAL: Always save media_key and parent_media_key separately. - # They are excluded from clean_attributes (to keep variant specs display clean) - # but MUST exist in variant_attributes for MEDIA_LINK to match and link images. + # Supports comma-separated keys as well as single keys. for mk_code, mk_name in (("media_key", "media_key"), ("parent_media_key", "parent_media_key")): raw_mk_val = all_attributes.get(mk_code) if raw_mk_val: - canonical_val = MediaResolver.canonicalize_media_key(str(raw_mk_val)) or str(raw_mk_val) - if canonical_val: - mk_attr_type = None - if attr_type_cache is not None and mk_code in attr_type_cache: - mk_attr_type = attr_type_cache[mk_code] - else: - mk_attr_type = db_session.query(AttributeType).filter(AttributeType.code == mk_code).first() - if not mk_attr_type: - mk_at_id = str(ulid.ULID()) - mk_attr_type = AttributeType( - attribute_id=mk_at_id, - name=mk_name, - code=mk_code, - status="active" - ) - db_session.add(mk_attr_type) - if attr_type_cache is not None: - attr_type_cache[mk_code] = mk_attr_type + # Split on commas for multi-image references + sub_keys = [k.strip() for k in str(raw_mk_val).split(",") if k.strip()] + if not sub_keys and str(raw_mk_val).strip(): + sub_keys = [str(raw_mk_val).strip()] - db_session.add(VariantAttribute( - id=str(ulid.ULID()), - variant_id=variant.variant_id, - attribute_id=mk_attr_type.attribute_id, - attribute_value=canonical_val - )) + mk_attr_type = None + if attr_type_cache is not None and mk_code in attr_type_cache: + mk_attr_type = attr_type_cache[mk_code] + else: + mk_attr_type = db_session.query(AttributeType).filter(AttributeType.code == mk_code).first() + if not mk_attr_type: + mk_at_id = str(ulid.ULID()) + mk_attr_type = AttributeType( + attribute_id=mk_at_id, + name=mk_name, + code=mk_code, + status="active" + ) + db_session.add(mk_attr_type) + if attr_type_cache is not None: + attr_type_cache[mk_code] = mk_attr_type - # 6. Attach Variant Images via Smart 3-Tier Canonical Media Key Resolver + for sk in sub_keys: + canonical_val = MediaResolver.canonicalize_media_key(sk) or sk + if canonical_val: + canonical_val = canonical_val[:2000] + db_session.add(VariantAttribute( + id=str(ulid.ULID()), + variant_id=variant.variant_id, + attribute_id=mk_attr_type.attribute_id, + attribute_value=canonical_val + )) + + # 6. Attach Variant Images via Smart Canonical Media Key Resolver (Supporting Comma-Separated Keys) raw_media_key = mapped_row.get("media_key") or mapped_row.get("parent_media_key") # Tier 2 Fallback: Match by SKU if no media_key column is in Excel @@ -588,24 +600,44 @@ class UpsertExecutor: if not raw_media_key: raw_media_key = mapped_row.get("name") or mapped_row.get("parent_name") - media_key = MediaResolver.canonicalize_media_key(raw_media_key) + if raw_media_key: + mk_tokens = [k.strip() for k in str(raw_media_key).split(",") if k.strip()] + if not mk_tokens and str(raw_media_key).strip(): + mk_tokens = [str(raw_media_key).strip()] - if media_key: - m_group = None - if media_group_cache is not None and media_key in media_group_cache: - m_group = media_group_cache[media_key] - else: - m_group = db_session.query(MediaGroup).filter(MediaGroup.media_key == media_key).first() - if not m_group and mapped_row.get("parent_media_key"): - p_key = MediaResolver.canonicalize_media_key(mapped_row.get("parent_media_key")) - if p_key: - m_group = db_session.query(MediaGroup).filter(MediaGroup.media_key == p_key).first() - if media_group_cache is not None: - media_group_cache[media_key] = m_group + all_matched_assets = [] + seen_asset_ids = set() - if m_group and m_group.media_assets: + for token_key in mk_tokens: + c_media_key = MediaResolver.canonicalize_media_key(token_key) + if not c_media_key: + continue + + m_group = None + if media_group_cache is not None and c_media_key in media_group_cache: + m_group = media_group_cache[c_media_key] + else: + m_group = db_session.query(MediaGroup).filter(MediaGroup.media_key == c_media_key).first() + if not m_group and mapped_row.get("parent_media_key"): + p_tokens = [pk.strip() for pk in str(mapped_row.get("parent_media_key")).split(",") if pk.strip()] + for p_tok in p_tokens: + p_key = MediaResolver.canonicalize_media_key(p_tok) + if p_key: + m_group = db_session.query(MediaGroup).filter(MediaGroup.media_key == p_key).first() + if m_group: + break + if media_group_cache is not None: + media_group_cache[c_media_key] = m_group + + if m_group and m_group.media_assets: + for asset in m_group.media_assets: + if asset.id not in seen_asset_ids: + seen_asset_ids.add(asset.id) + all_matched_assets.append(asset) + + if all_matched_assets: db_session.query(VariantImage).filter(VariantImage.variant_id == variant.variant_id).delete() - for idx, asset in enumerate(m_group.media_assets): + for idx, asset in enumerate(all_matched_assets): vi_id = str(ulid.ULID()) var_img = VariantImage( image_id=vi_id, @@ -770,6 +802,13 @@ class UpsertExecutor: sp.rollback() except Exception: pass + # Evict uncommitted series and models from cache to prevent cascading foreign key failures + if series_cache is not None: + for sk in new_series_keys: + series_cache.pop(sk, None) + if model_cache is not None: + for mk in new_model_keys: + model_cache.pop(mk, None) fail_count += 1 runtime_errors.append({ "row_number": row_number,