90 lines
3.4 KiB
Python
90 lines
3.4 KiB
Python
import csv
|
|
import io
|
|
import os
|
|
from typing import Generator, List, Dict, Any, Tuple
|
|
import openpyxl
|
|
|
|
class DataFileParser:
|
|
"""
|
|
Parser for CSV, XLSX, and ODS tabular data files.
|
|
Supports streaming generation of rows to minimize RAM usage on large imports (100,000+ rows).
|
|
"""
|
|
|
|
@classmethod
|
|
def _detect_csv_encoding(cls, file_path: str) -> str:
|
|
"""
|
|
Detects appropriate encoding for CSV files, handling Excel CP1252/Windows-1252 gracefully.
|
|
"""
|
|
try:
|
|
with open(file_path, "rb") as bf:
|
|
sample = bf.read(131072)
|
|
try:
|
|
sample.decode("utf-8")
|
|
return "utf-8-sig"
|
|
except UnicodeDecodeError:
|
|
try:
|
|
sample.decode("cp1252")
|
|
return "cp1252"
|
|
except UnicodeDecodeError:
|
|
return "latin1"
|
|
except Exception:
|
|
return "utf-8-sig"
|
|
|
|
@classmethod
|
|
def get_headers(cls, file_path: str, file_format: str) -> List[str]:
|
|
"""
|
|
Extracts column headers from file without parsing full content.
|
|
"""
|
|
fmt = file_format.upper()
|
|
if fmt == "CSV":
|
|
enc = cls._detect_csv_encoding(file_path)
|
|
with open(file_path, "r", encoding=enc, errors="replace") as f:
|
|
reader = csv.reader(f)
|
|
for row in reader:
|
|
return [c.strip() for c in row if c is not None]
|
|
elif fmt in ("XLSX", "ODS"):
|
|
wb = openpyxl.load_workbook(file_path, read_only=True, data_only=True)
|
|
sheet = wb.active
|
|
for row in sheet.iter_rows(values_only=True):
|
|
wb.close()
|
|
return [str(c).strip() for c in row if c is not None]
|
|
wb.close()
|
|
return []
|
|
|
|
@classmethod
|
|
def stream_rows(cls, file_path: str, file_format: str) -> Generator[Tuple[int, Dict[str, Any]], None, None]:
|
|
"""
|
|
Yields (row_number, row_dict) tuples line-by-line.
|
|
"""
|
|
fmt = file_format.upper()
|
|
if fmt == "CSV":
|
|
enc = cls._detect_csv_encoding(file_path)
|
|
with open(file_path, "r", encoding=enc, errors="replace") as f:
|
|
reader = csv.DictReader(f)
|
|
row_idx = 2 # Row 1 is headers
|
|
for row in reader:
|
|
cleaned_row = {str(k).strip(): str(v).strip() if v is not None else "" for k, v in row.items() if k}
|
|
yield row_idx, cleaned_row
|
|
row_idx += 1
|
|
|
|
elif fmt in ("XLSX", "ODS"):
|
|
wb = openpyxl.load_workbook(file_path, read_only=True, data_only=True)
|
|
sheet = wb.active
|
|
headers = []
|
|
row_idx = 1
|
|
for row in sheet.iter_rows(values_only=True):
|
|
if row_idx == 1:
|
|
headers = [str(c).strip() if c is not None else f"Column_{i}" for i, c in enumerate(row)]
|
|
else:
|
|
row_dict = {}
|
|
has_data = False
|
|
for i, cell_val in enumerate(row):
|
|
if i < len(headers):
|
|
val_str = str(cell_val).strip() if cell_val is not None else ""
|
|
row_dict[headers[i]] = val_str
|
|
if val_str:
|
|
has_data = True
|
|
if has_data:
|
|
yield row_idx, row_dict
|
|
row_idx += 1
|
|
wb.close()
|