86 lines
2.5 KiB
Python
86 lines
2.5 KiB
Python
"""Backfill houses.address_fingerprint for legacy rows.
|
|
|
|
Usage:
|
|
cd tradein-mvp/backend
|
|
DATABASE_URL=postgresql://... uv run python ../scripts/backfill_001_address_fingerprints.py
|
|
|
|
Idempotent -- safe to re-run. Updates only rows with NULL address_fingerprint.
|
|
Logs progress every 100 rows.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import sys
|
|
|
|
import psycopg
|
|
|
|
# Allow running from repo root via `python tradein-mvp/scripts/...`
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "backend"))
|
|
|
|
from app.services.matching.normalize import (
|
|
address_fingerprint,
|
|
normalize_address,
|
|
)
|
|
|
|
logging.basicConfig(
|
|
format="%(asctime)s [%(levelname)s] %(message)s",
|
|
level=logging.INFO,
|
|
)
|
|
logger = logging.getLogger("backfill_001")
|
|
|
|
|
|
def main() -> int:
|
|
dsn = os.environ.get("DATABASE_URL")
|
|
if not dsn:
|
|
logger.error("DATABASE_URL env var required")
|
|
return 1
|
|
|
|
# Strip SQLAlchemy dialect prefix if present (tradein-mvp uses postgresql+psycopg://)
|
|
dsn = dsn.replace("postgresql+psycopg://", "postgresql://", 1)
|
|
|
|
updated = 0
|
|
skipped = 0
|
|
with psycopg.connect(dsn) as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"SELECT id, address, lat, lon FROM houses "
|
|
"WHERE address_fingerprint IS NULL AND address IS NOT NULL"
|
|
)
|
|
rows = cur.fetchall()
|
|
logger.info("Found %d houses missing fingerprint", len(rows))
|
|
|
|
for i, (house_id, address, lat, lon) in enumerate(rows, 1):
|
|
normalized = normalize_address(address)
|
|
if normalized is None:
|
|
skipped += 1
|
|
continue
|
|
fp = address_fingerprint(normalized, lat, lon)
|
|
if fp is None:
|
|
skipped += 1
|
|
continue
|
|
cur.execute(
|
|
"UPDATE houses SET address_fingerprint = %s WHERE id = %s",
|
|
(fp, house_id),
|
|
)
|
|
updated += 1
|
|
if i % 100 == 0:
|
|
conn.commit()
|
|
logger.info(
|
|
"Progress: %d/%d updated=%d skipped=%d",
|
|
i,
|
|
len(rows),
|
|
updated,
|
|
skipped,
|
|
)
|
|
|
|
conn.commit()
|
|
|
|
logger.info(
|
|
"Done: updated=%d skipped=%d total=%d", updated, skipped, updated + skipped
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|