68 lines
2.7 KiB
Python
68 lines
2.7 KiB
Python
import csv
|
|
import io
|
|
import os
|
|
from typing import Generator, List, Dict, Any, Tuple
|
|
import openpyxl
|
|
|
|
class DataFileParser:
|
|
"""
|
|
Parser for CSV, XLSX, and ODS tabular data files.
|
|
Supports streaming generation of rows to minimize RAM usage on large imports (100,000+ rows).
|
|
"""
|
|
|
|
@classmethod
|
|
def get_headers(cls, file_path: str, file_format: str) -> List[str]:
|
|
"""
|
|
Extracts column headers from file without parsing full content.
|
|
"""
|
|
fmt = file_format.upper()
|
|
if fmt == "CSV":
|
|
with open(file_path, "r", encoding="utf-8-sig") as f:
|
|
reader = csv.reader(f)
|
|
for row in reader:
|
|
return [c.strip() for c in row if c is not None]
|
|
elif fmt in ("XLSX", "ODS"):
|
|
wb = openpyxl.load_workbook(file_path, read_only=True, data_only=True)
|
|
sheet = wb.active
|
|
for row in sheet.iter_rows(values_only=True):
|
|
wb.close()
|
|
return [str(c).strip() for c in row if c is not None]
|
|
wb.close()
|
|
return []
|
|
|
|
@classmethod
|
|
def stream_rows(cls, file_path: str, file_format: str) -> Generator[Tuple[int, Dict[str, Any]], None, None]:
|
|
"""
|
|
Yields (row_number, row_dict) tuples line-by-line.
|
|
"""
|
|
fmt = file_format.upper()
|
|
if fmt == "CSV":
|
|
with open(file_path, "r", encoding="utf-8-sig") as f:
|
|
reader = csv.DictReader(f)
|
|
row_idx = 2 # Row 1 is headers
|
|
for row in reader:
|
|
cleaned_row = {str(k).strip(): str(v).strip() if v is not None else "" for k, v in row.items() if k}
|
|
yield row_idx, cleaned_row
|
|
row_idx += 1
|
|
|
|
elif fmt in ("XLSX", "ODS"):
|
|
wb = openpyxl.load_workbook(file_path, read_only=True, data_only=True)
|
|
sheet = wb.active
|
|
headers = []
|
|
row_idx = 1
|
|
for row in sheet.iter_rows(values_only=True):
|
|
if row_idx == 1:
|
|
headers = [str(c).strip() if c is not None else f"Column_{i}" for i, c in enumerate(row)]
|
|
else:
|
|
row_dict = {}
|
|
has_data = False
|
|
for i, cell_val in enumerate(row):
|
|
if i < len(headers):
|
|
val_str = str(cell_val).strip() if cell_val is not None else ""
|
|
row_dict[headers[i]] = val_str
|
|
if val_str:
|
|
has_data = True
|
|
if has_data:
|
|
yield row_idx, row_dict
|
|
row_idx += 1
|
|
wb.close()
|