433 lines
15 KiB
Python
433 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Provision an isolated simulation copy of the IfixKart MySQL schemas.
|
|
|
|
What it does
|
|
------------
|
|
1. Creates two dedicated databases on the same MySQL server:
|
|
<commerce>_sim (catalog + ecommerce + CRM tables)
|
|
<admin>_sim (core/admin tables)
|
|
2. Clones every table *structure* from the source schemas.
|
|
3. Copies only safe reference/catalog data (never customers, orders, payments).
|
|
4. Seeds deterministic stock receipts so agents can actually check out.
|
|
5. Grants the normal application DB user access to the new schemas.
|
|
6. Writes Backend/.env.sim pointing the app at the sim schemas.
|
|
|
|
Requires root SSH access to the MySQL host (the same host used by ssh_tunnel.py)
|
|
because the application DB user is intentionally not allowed to CREATE DATABASE.
|
|
|
|
Usage
|
|
-----
|
|
cd Backend
|
|
./venv/bin/python scripts/sim/provision_sim_db.py --recreate
|
|
./venv/bin/python scripts/sim/provision_sim_db.py --recreate --stock-per-variant 5000
|
|
|
|
After provisioning:
|
|
./venv/bin/python scripts/sim/seed_customers.py --count 10000
|
|
./venv/bin/python scripts/sim/run_sim_backend.py
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import shlex
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
import paramiko
|
|
import ulid
|
|
from dotenv import load_dotenv
|
|
from sqlalchemy import create_engine, text
|
|
from sqlalchemy.engine import make_url
|
|
|
|
BACKEND_DIR = Path(__file__).resolve().parents[2]
|
|
ROOT_DIR = BACKEND_DIR.parent
|
|
ENV_FILE = BACKEND_DIR / ".env"
|
|
SIM_ENV_FILE = BACKEND_DIR / ".env.sim"
|
|
SIM_STATE_DIR = ROOT_DIR / ".sim"
|
|
|
|
CORE_COPY_TABLES = [
|
|
"roles",
|
|
"permissions",
|
|
"departments",
|
|
"designations",
|
|
"settings",
|
|
"countries",
|
|
"states",
|
|
"cities",
|
|
"notification_templates",
|
|
"role_permissions",
|
|
"user_location_mappings",
|
|
"users",
|
|
"file_uploads",
|
|
]
|
|
|
|
COMMERCE_COPY_TABLES = [
|
|
# Catalog / storefront reference data
|
|
"attribute_types",
|
|
"brands",
|
|
"categories",
|
|
"device_series",
|
|
"device_models",
|
|
"device_type_registry",
|
|
"brand_device_types",
|
|
"products",
|
|
"product_variants",
|
|
"product_images",
|
|
"variant_images",
|
|
"variant_attributes",
|
|
"tags",
|
|
"product_tags",
|
|
"collections",
|
|
"product_collections",
|
|
"seo_metadata",
|
|
"storefront_settings",
|
|
"storefront_footer_info",
|
|
"storefront_contents",
|
|
"storefront_catalog_filters",
|
|
"storefront_mega_menu",
|
|
"service_catalog",
|
|
# Lightweight inventory/location reference data (all empty in prod today)
|
|
"crm_locations",
|
|
"crm_warehouses",
|
|
"crm_suppliers",
|
|
"product_locations",
|
|
]
|
|
|
|
|
|
def log(msg: str) -> None:
|
|
print(f"[provision] {msg}", flush=True)
|
|
|
|
|
|
def load_ssh_credentials() -> tuple[str, str, str]:
|
|
"""Read SSH host/user/password from ssh_tunnel.py (already part of this repo)."""
|
|
host = os.getenv("SIM_SSH_HOST", "")
|
|
user = os.getenv("SIM_SSH_USER", "")
|
|
password = os.getenv("SIM_SSH_PASS", "")
|
|
if host and user and password:
|
|
return host, user, password
|
|
|
|
tunnel = BACKEND_DIR / "ssh_tunnel.py"
|
|
if not tunnel.exists():
|
|
raise SystemExit("ssh_tunnel.py not found and SIM_SSH_* env vars not set")
|
|
src = tunnel.read_text()
|
|
host = re.search(r'SSH_HOST\s*=\s*"([^"]+)"', src).group(1)
|
|
user = re.search(r'SSH_USER\s*=\s*"([^"]+)"', src).group(1)
|
|
password = re.search(r'SSH_PASS\s*=\s*"([^"]+)"', src).group(1)
|
|
return host, user, password
|
|
|
|
|
|
class MySQLAdmin:
|
|
def __init__(self, app_user: str | None = None, app_password: str | None = None) -> None:
|
|
host, user, password = load_ssh_credentials()
|
|
self.host = host
|
|
self.app_user = app_user
|
|
self.app_password = app_password
|
|
self.client = paramiko.SSHClient()
|
|
self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
self.client.connect(host, 22, user, password, timeout=20)
|
|
|
|
def close(self) -> None:
|
|
self.client.close()
|
|
|
|
def _run(self, command: str, sql: str) -> str:
|
|
stdin, stdout, stderr = self.client.exec_command(command, timeout=1800)
|
|
stdin.write(sql)
|
|
stdin.channel.shutdown_write()
|
|
out = stdout.read().decode(errors="replace")
|
|
err = stderr.read().decode(errors="replace")
|
|
status = stdout.channel.recv_exit_status()
|
|
if status != 0:
|
|
raise RuntimeError(f"MySQL command failed ({status}):\n{err[-4000:]}")
|
|
return out + err
|
|
|
|
def exec_sql(self, sql: str, *, check: bool = True) -> str:
|
|
try:
|
|
return self._run("mysql -uroot -N --batch --raw", sql)
|
|
except RuntimeError:
|
|
if check:
|
|
raise
|
|
return ""
|
|
|
|
def exec_as_app(self, sql: str) -> str:
|
|
"""Run SQL as the application user on localhost (has GRANT OPTION here)."""
|
|
if not self.app_user or not self.app_password:
|
|
raise RuntimeError("Application DB credentials unavailable for GRANT")
|
|
command = (
|
|
f"MYSQL_PWD={shlex.quote(self.app_password)} "
|
|
f"mysql -u{shlex.quote(self.app_user)} -N --batch --raw"
|
|
)
|
|
return self._run(command, sql)
|
|
|
|
def query(self, sql: str) -> list[list[str]]:
|
|
raw = self.exec_sql(sql + ";\n")
|
|
rows: list[list[str]] = []
|
|
for line in raw.splitlines():
|
|
if not line.strip():
|
|
continue
|
|
rows.append(line.split("\t"))
|
|
return rows
|
|
|
|
|
|
def qident(name: str) -> str:
|
|
return "`" + name.replace("`", "``") + "`"
|
|
|
|
|
|
def qstr(value: str) -> str:
|
|
return "'" + value.replace("\\", "\\\\").replace("'", "''") + "'"
|
|
|
|
|
|
def sim_database_url(source_url: str, suffix: str) -> str:
|
|
url = make_url(source_url)
|
|
new_name = f"{url.database}{suffix}"
|
|
# render_as_string is required: str(URL) deliberately masks the password.
|
|
return url.set(database=new_name).render_as_string(hide_password=False)
|
|
|
|
|
|
def db_name(url: str) -> str:
|
|
parsed = make_url(url)
|
|
if not parsed.database:
|
|
raise SystemExit(f"Database URL has no database name: {url}")
|
|
return parsed.database
|
|
|
|
|
|
def copy_tables(admin: MySQLAdmin, source_db: str, target_db: str, tables: list[str] | None = None) -> list[str]:
|
|
if tables is None:
|
|
rows = admin.query(
|
|
"SELECT table_name FROM information_schema.tables "
|
|
f"WHERE table_schema={qstr(source_db)} AND table_type='BASE TABLE' "
|
|
"ORDER BY table_name"
|
|
)
|
|
tables = [r[0] for r in rows]
|
|
|
|
stmts = ["SET FOREIGN_KEY_CHECKS=0;"]
|
|
for table in tables:
|
|
stmts.append(
|
|
f"CREATE TABLE {qident(target_db)}.{qident(table)} "
|
|
f"LIKE {qident(source_db)}.{qident(table)};"
|
|
)
|
|
stmts.append("SET FOREIGN_KEY_CHECKS=1;")
|
|
admin.exec_sql("\n".join(stmts))
|
|
return tables
|
|
|
|
|
|
def copy_data(admin: MySQLAdmin, source_db: str, target_db: str, tables: list[str]) -> None:
|
|
stmts = ["SET FOREIGN_KEY_CHECKS=0;"]
|
|
for table in tables:
|
|
stmts.append(
|
|
f"INSERT INTO {qident(target_db)}.{qident(table)} "
|
|
f"SELECT * FROM {qident(source_db)}.{qident(table)};"
|
|
)
|
|
stmts.append("SET FOREIGN_KEY_CHECKS=1;")
|
|
admin.exec_sql("\n".join(stmts))
|
|
|
|
|
|
def seed_inventory(commerce_url: str, stock_per_variant: int) -> int:
|
|
engine = create_engine(commerce_url, future=True)
|
|
seeded = 0
|
|
with engine.begin() as conn:
|
|
exists = conn.execute(
|
|
text("SELECT COUNT(*) FROM information_schema.tables "
|
|
"WHERE table_schema=DATABASE() AND table_name='inventory_ledger'")
|
|
).scalar()
|
|
if not exists:
|
|
raise RuntimeError("inventory_ledger table missing in simulation schema")
|
|
conn.execute(text("DELETE FROM inventory_ledger"))
|
|
variants = [r[0] for r in conn.execute(text("SELECT variant_id FROM product_variants"))]
|
|
now = datetime.now(timezone.utc).replace(tzinfo=None)
|
|
rows = [
|
|
{
|
|
"ledger_id": str(ulid.ULID()),
|
|
"variant_id": variant_id,
|
|
"event_type": "RECEIPT",
|
|
"qty": stock_per_variant,
|
|
"reference_id": "SIM_SEED",
|
|
"notes": "Simulation opening stock",
|
|
"created_at": now,
|
|
}
|
|
for variant_id in variants
|
|
]
|
|
if rows:
|
|
conn.execute(
|
|
text(
|
|
"INSERT INTO inventory_ledger "
|
|
"(ledger_id, variant_id, event_type, qty, reference_id, notes, created_at) "
|
|
"VALUES (:ledger_id, :variant_id, :event_type, :qty, :reference_id, :notes, :created_at)"
|
|
),
|
|
rows,
|
|
)
|
|
seeded = len(rows)
|
|
return seeded
|
|
|
|
|
|
def write_sim_env(
|
|
core_url: str,
|
|
commerce_url: str,
|
|
crm_url: str,
|
|
email_domain: str,
|
|
) -> None:
|
|
# Keep the same credentials/query parameters as production, only swap the DB.
|
|
lines = [
|
|
"# Generated by scripts/sim/provision_sim_db.py -- do not commit.",
|
|
"ENVIRONMENT=simulation",
|
|
"RUNTIME_ENV=simulation",
|
|
"SIMULATION_MODE=true",
|
|
"PAYMENT_SIMULATOR_ENABLED=true",
|
|
f"SIM_CUSTOMER_EMAIL_DOMAIN={email_domain}",
|
|
f"DATABASE_URL={core_url}",
|
|
f"CORE_DATABASE_URL={core_url}",
|
|
f"COMMERCE_DATABASE_URL={commerce_url}",
|
|
f"CRM_DATABASE_URL={crm_url}",
|
|
"REDIS_URL=redis://127.0.0.1:6379/3",
|
|
"RAZORPAY_ENABLED=false",
|
|
"SHIPROCKET_ENABLED=false",
|
|
"WATI_ENABLED=false",
|
|
"KNOWLARITY_ENABLED=false",
|
|
"GA4_ENABLED=false",
|
|
"META_ENABLED=false",
|
|
"SIM_THINK_TIME_SCALE=0.05",
|
|
"",
|
|
]
|
|
SIM_ENV_FILE.write_text("\n".join(lines))
|
|
try:
|
|
SIM_ENV_FILE.chmod(0o600)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def ensure_gitignore() -> None:
|
|
gi = ROOT_DIR / ".gitignore"
|
|
entries = [".env.sim", ".sim/", "Backend/sim-artifacts/"]
|
|
existing = gi.read_text().splitlines() if gi.exists() else []
|
|
missing = [e for e in entries if e not in existing]
|
|
if missing:
|
|
with gi.open("a") as f:
|
|
f.write("\n# Simulation runtime (generated, may contain credentials)\n")
|
|
for e in missing:
|
|
f.write(e + "\n")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Provision isolated IfixKart simulation schemas")
|
|
parser.add_argument("--suffix", default="_sim", help="Database name suffix (default: _sim)")
|
|
parser.add_argument("--recreate", action="store_true", help="Drop the simulation DBs first")
|
|
parser.add_argument("--stock-per-variant", type=int, default=5000, help="Opening stock per active variant")
|
|
parser.add_argument("--email-domain", default="@sim.example.com", help="Email domain for simulated customers")
|
|
parser.add_argument("--skip-env", action="store_true", help="Do not write Backend/.env.sim")
|
|
args = parser.parse_args()
|
|
|
|
if not ENV_FILE.exists():
|
|
raise SystemExit(f"Missing {ENV_FILE}")
|
|
load_dotenv(ENV_FILE, override=False)
|
|
|
|
core_source = os.getenv("CORE_DATABASE_URL") or os.getenv("DATABASE_URL")
|
|
commerce_source = os.getenv("COMMERCE_DATABASE_URL") or os.getenv("DATABASE_URL")
|
|
crm_source = os.getenv("CRM_DATABASE_URL") or commerce_source
|
|
if not core_source or not commerce_source:
|
|
raise SystemExit("CORE_DATABASE_URL / COMMERCE_DATABASE_URL are not configured in .env")
|
|
|
|
core_sim = sim_database_url(core_source, args.suffix)
|
|
commerce_sim = sim_database_url(commerce_source, args.suffix)
|
|
crm_sim = sim_database_url(crm_source, args.suffix)
|
|
|
|
core_src_db = db_name(core_source)
|
|
commerce_src_db = db_name(commerce_source)
|
|
crm_src_db = db_name(crm_source)
|
|
core_sim_db = db_name(core_sim)
|
|
commerce_sim_db = db_name(commerce_sim)
|
|
crm_sim_db = db_name(crm_sim)
|
|
|
|
app_user = make_url(commerce_source).username
|
|
app_password = make_url(commerce_source).password
|
|
if not app_user:
|
|
raise SystemExit("Could not read the application DB username from .env")
|
|
|
|
admin = MySQLAdmin(app_user=app_user, app_password=app_password)
|
|
try:
|
|
log(f"Connected to MySQL host {admin.host} as admin")
|
|
if args.recreate:
|
|
log(f"Dropping {core_sim_db}, {commerce_sim_db}, {crm_sim_db}")
|
|
admin.exec_sql(
|
|
f"DROP DATABASE IF EXISTS {qident(core_sim_db)};\n"
|
|
f"DROP DATABASE IF EXISTS {qident(commerce_sim_db)};\n"
|
|
f"DROP DATABASE IF EXISTS {qident(crm_sim_db)};\n"
|
|
)
|
|
|
|
for name in (core_sim_db, commerce_sim_db, crm_sim_db):
|
|
existing = admin.query(
|
|
"SELECT COUNT(*) FROM information_schema.tables "
|
|
f"WHERE table_schema={qstr(name)}"
|
|
)
|
|
count = int(existing[0][0]) if existing else 0
|
|
if count:
|
|
raise SystemExit(
|
|
f"Simulation database {name} already has {count} tables. "
|
|
"Re-run with --recreate to rebuild it."
|
|
)
|
|
admin.exec_sql(
|
|
f"CREATE DATABASE IF NOT EXISTS {qident(name)} "
|
|
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
|
|
)
|
|
|
|
log("Cloning table structures")
|
|
core_tables = copy_tables(admin, core_src_db, core_sim_db)
|
|
commerce_tables = copy_tables(admin, commerce_src_db, commerce_sim_db)
|
|
crm_tables = copy_tables(admin, crm_src_db, crm_sim_db)
|
|
log(
|
|
f" core: {len(core_tables)} tables, commerce: {len(commerce_tables)} tables, "
|
|
f"crm: {len(crm_tables)} tables"
|
|
)
|
|
|
|
log("Copying reference/catalog data")
|
|
core_copy = [t for t in CORE_COPY_TABLES if t in core_tables]
|
|
commerce_copy = [t for t in COMMERCE_COPY_TABLES if t in commerce_tables]
|
|
copy_data(admin, core_src_db, core_sim_db, core_copy)
|
|
copy_data(admin, commerce_src_db, commerce_sim_db, commerce_copy)
|
|
# The CRM schema is small (services/repair catalog only). Copy it entirely.
|
|
copy_data(admin, crm_src_db, crm_sim_db, crm_tables)
|
|
log(f" copied {len(core_copy)} core + {len(commerce_copy)} commerce + {len(crm_tables)} crm tables")
|
|
|
|
log(f"Granting {app_user} access to simulation schemas")
|
|
for name in (core_sim_db, commerce_sim_db, crm_sim_db):
|
|
admin.exec_as_app(
|
|
f"GRANT ALL PRIVILEGES ON {qident(name)}.* TO "
|
|
f"{qstr(app_user)}@'127.0.0.1';\nFLUSH PRIVILEGES;\n"
|
|
)
|
|
finally:
|
|
admin.close()
|
|
|
|
log("Seeding opening stock")
|
|
seeded = seed_inventory(commerce_sim, args.stock_per_variant)
|
|
log(f" {seeded} variants x {args.stock_per_variant} units")
|
|
|
|
ensure_gitignore()
|
|
if not args.skip_env:
|
|
write_sim_env(core_sim, commerce_sim, crm_sim, args.email_domain)
|
|
log(f"Wrote {SIM_ENV_FILE}")
|
|
|
|
SIM_STATE_DIR.mkdir(parents=True, exist_ok=True)
|
|
state = {
|
|
"provisioned_at": datetime.now(timezone.utc).isoformat(),
|
|
"suffix": args.suffix,
|
|
"core_db": core_sim_db,
|
|
"commerce_db": commerce_sim_db,
|
|
"crm_db": crm_sim_db,
|
|
"core_source": core_src_db,
|
|
"commerce_source": commerce_src_db,
|
|
"crm_source": crm_src_db,
|
|
"stock_per_variant": args.stock_per_variant,
|
|
"core_tables": len(core_tables),
|
|
"commerce_tables": len(commerce_tables),
|
|
"crm_tables": len(crm_tables),
|
|
"catalog_rows": {t: None for t in commerce_copy},
|
|
}
|
|
(SIM_STATE_DIR / "provision.json").write_text(json.dumps(state, indent=2))
|
|
log(f"Done. State: {SIM_STATE_DIR / 'provision.json'}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|