gendesign/tradein-mvp/backend/app/tasks/cadastral_geo_match.py

"""Geo-nearest cadastral building matcher for listings (#cadastral-geo-match).

Fills `listings.building_cadastral_number` (currently 0% / ~43k rows) from the nearest
cadastral building, via a LOCAL materialized mirror of the gendesign_cad_buildings FDW.

WHY a local mirror (perf fact, measured live):
    gendesign_cad_buildings is a postgres_fdw foreign table (~36.7k EKB rows) with NO
    PostGIS geom — only scalar lat/lon. A per-listing nearest-building query over the FDW
    takes ~1.16s/row → ~13h for 43k listings. UNUSABLE per-row.
    ⇒ We materialize the FDW once (single bulk scan) into cad_buildings_local with a real
      Point geom + GIST, then run a fast LOCAL KNN nearest-neighbour join (geom <-> point).
    The whole local backfill runs in well under a minute (see PR EXPLAIN evidence).

APPROXIMATION (deliberate first increment):
    This is a GEO-NEAREST match — a street-level-geocoded listing is matched to the nearest
    cadastral building within `threshold_m`, NOT necessarily its exact cadastral building.
    The threshold is always logged. Exact cadastral resolution + parcel-containment are
    deferred (cad_parcels FDW not exposed). Tier-0 house matching in the estimator already
    treats building_cadastral_number as a hint, not ground truth, so an approximate fill is
    a net win over 0% coverage.

Pipeline (one combined run, scheduler source='cadastral_geo_match'):
    1. refresh_cad_buildings_local(db) — TRUNCATE + bulk INSERT from FDW (one scan).
    2. match_listings_to_buildings(db, threshold_m) — set-based LATERAL KNN UPDATE.
    Refresh-then-match in ONE run guarantees the match always reads a fresh mirror.

psycopg v3: app SQL uses CAST(:x AS type), never :x::type.
"""

from __future__ import annotations

import logging
import math
import time
from dataclasses import dataclass, field

from sqlalchemy import text
from sqlalchemy.orm import Session

from app.services import scrape_runs as runs_mod

logger = logging.getLogger(__name__)

DEFAULT_THRESHOLD_M = 50
DEFAULT_BATCH_SIZE = 5000

# EKB latitude — used to convert the metric threshold into a degree gate radius for the
# GIST-indexable geometry-space pre-filter (the gate is a SUPERSET of the true circle; the
# ST_DistanceSphere recheck decides acceptance, so a slightly-too-wide gate is harmless).
_EKB_LAT_DEG = 56.84
_M_PER_DEG_LAT = 111_320.0


def _deg_gate_for(threshold_m: float) -> float:
    """Degree radius that always encloses `threshold_m` metres at EKB latitude.

    Longitude degrees shrink by cos(lat), so we size the gate on the tighter (longitude)
    axis and apply a 1.5× safety factor. The gate only pre-filters candidates for the GIST
    KNN scan; the exact ST_DistanceSphere(...) <= threshold_m recheck is authoritative, so
    over-sizing the gate never produces a false match, only a few extra candidates to rank.
    """
    m_per_deg_lon = _M_PER_DEG_LAT * math.cos(math.radians(_EKB_LAT_DEG))
    return 1.5 * threshold_m / m_per_deg_lon


# ── Refresh: materialize FDW → cad_buildings_local (single bulk scan) ─────────
def refresh_cad_buildings_local(db: Session) -> int:
    """TRUNCATE cad_buildings_local; bulk INSERT from gendesign_cad_buildings FDW.

    Single FDW scan (no per-row round-trips). Builds Point(4326) geom from lon/lat.
    Idempotent: TRUNCATE+INSERT in one transaction — the table is fully replaced atomically.

    Returns the number of rows loaded.
    """
    db.execute(text("TRUNCATE cad_buildings_local"))
    db.execute(
        text(
            """
            INSERT INTO cad_buildings_local (
                cad_num, readable_address, year_built, floors, area_m2, purpose,
                lat, lon, geom
            )
            SELECT
                cad_num, readable_address, year_built, floors, area_m2, purpose,
                lat, lon,
                ST_SetSRID(ST_MakePoint(lon, lat), 4326)
            FROM gendesign_cad_buildings
            WHERE lat IS NOT NULL
              AND lon IS NOT NULL
              AND cad_num IS NOT NULL
            ON CONFLICT (cad_num) DO NOTHING
            """
        )
    )
    count = db.execute(text("SELECT count(*) FROM cad_buildings_local")).scalar() or 0
    db.commit()
    logger.info("refresh_cad_buildings_local: loaded %d rows from FDW", count)
    return int(count)


# ── Match: LATERAL KNN nearest building → listings.building_cadastral_number ──
# Set-based UPDATE. For each candidate listing (geo present, optionally only NULL bcn) the
# LATERAL subquery picks the single nearest cad_buildings_local building and accepts it only
# if its TRUE distance is within :threshold_m metres.
#
# PERF (measured live, the whole point of this design):
#   The gate is a GEOMETRY-space ST_DWithin(cb.geom, l_geom, :deg_gate) — planar degrees,
#   fully GIST-indexable, NO per-row geography cast. The `<-> ` KNN order then returns the
#   single nearest candidate via the GIST index. ONLY that one nearest row is converted to a
#   true metric distance via ST_DistanceSphere (spherical metres) and compared to :threshold_m.
#   This is ~175× faster than a geography-cast ST_DWithin in the WHERE: a geography filter
#   forced a per-candidate cast+recheck that the planner could not push into the index, so the
#   full 41k-listing UPDATE ran 58s+ and never finished; the geometry-gate version does a
#   5000-row chunk in ~330ms and the full ~41k backfill in a few seconds.
#
#   :deg_gate is a degree radius that ALWAYS encloses :threshold_m at EKB latitude (~56.8°):
#   1° lat ≈ 111_320 m, and we widen by /cos(lat) for longitude + a safety factor, so the
#   gate is a superset of the true threshold circle and the ST_DistanceSphere recheck is what
#   actually decides acceptance (no false negatives from the gate).
#
# Batching: pick a chunk of candidate listing ids first (LIMIT :batch_size), then run the
# LATERAL UPDATE only over that chunk — bounds lock/WAL footprint on the ~43k table and lets
# the run loop checkpoint a heartbeat between chunks. only_missing=true makes each chunk drain
# monotonically (filled rows drop out of the candidate set), so the loop terminates.
# Single set-based LATERAL KNN UPDATE over ALL candidates in one statement.
# NOT chunked by OFFSET: an OFFSET walk over the `building_cadastral_number IS NULL`
# predicate is unsound because the UPDATE mutates that very predicate mid-walk —
# matched rows drop out of the candidate set, so OFFSET (reset OR advance) skips
# un-processed rows. The full UPDATE is ~5s for ~41k listings (GIST KNN, measured),
# so a single statement is both correct and fast enough; no chunking needed.
_MATCH_SQL = text(
    """
    WITH matched AS (
        SELECT l.id AS id, m.cad_num AS cad_num
        FROM listings l
        JOIN LATERAL (
            SELECT cb.cad_num,
                   ST_DistanceSphere(cb.geom, l.geom) AS dist_m
            FROM cad_buildings_local cb
            WHERE ST_DWithin(cb.geom, l.geom, CAST(:deg_gate AS double precision))
            ORDER BY cb.geom <-> l.geom
            LIMIT 1
        ) m ON true
        WHERE l.geom IS NOT NULL
          AND (
              CAST(:only_missing AS boolean) = false
              OR l.building_cadastral_number IS NULL
          )
          AND m.dist_m <= CAST(:threshold_m AS double precision)
    )
    UPDATE listings l
    SET building_cadastral_number = matched.cad_num
    FROM matched
    WHERE l.id = matched.id
    """
)

_CANDIDATES_SQL = text(
    """
    SELECT count(*) FROM listings l
    WHERE l.geom IS NOT NULL
      AND (CAST(:only_missing AS boolean) = false OR l.building_cadastral_number IS NULL)
    """
)


@dataclass
class CadMatchResult:
    buildings_loaded: int = 0  # rows in cad_buildings_local after refresh
    candidates_total: int = 0  # listings eligible for matching (geo present + filter)
    listings_matched: int = 0  # listings updated with a building_cadastral_number
    threshold_m: int = DEFAULT_THRESHOLD_M
    duration_sec: float = field(default=0.0)

    def to_counters(self) -> dict[str, int]:
        return {
            "buildings_loaded": self.buildings_loaded,
            "candidates_total": self.candidates_total,
            "listings_matched": self.listings_matched,
            "threshold_m": self.threshold_m,
            "duration_sec": int(self.duration_sec),
        }


def match_listings_to_buildings(
    db: Session,
    *,
    threshold_m: int = DEFAULT_THRESHOLD_M,
    batch_size: int = DEFAULT_BATCH_SIZE,
    only_missing: bool = True,
) -> int:
    """Set-based LATERAL KNN UPDATE: fill listings.building_cadastral_number.

    GEO-NEAREST (approximate): each candidate listing is matched to the nearest
    cad_buildings_local building within `threshold_m` metres. The threshold is logged.

    Single set-based UPDATE over ALL candidates (no chunking). An OFFSET walk would be
    unsound: the UPDATE mutates the `building_cadastral_number IS NULL` predicate it pages
    over, so matched rows drop out and OFFSET (reset OR advance) skips un-processed rows.
    The full UPDATE is ~5s for ~41k listings (GIST KNN), so one statement is correct and
    fast. `batch_size` is accepted for schedule-param compatibility but no longer used.
    Returns listings matched.
    """
    logger.info(
        "match_listings_to_buildings: GEO-NEAREST match threshold_m=%d "
        "only_missing=%s (approximate: nearest building, not exact cadastral)",
        threshold_m,
        only_missing,
    )

    deg_gate = _deg_gate_for(float(threshold_m))
    result = db.execute(
        _MATCH_SQL,
        {
            "threshold_m": float(threshold_m),
            "deg_gate": deg_gate,
            "only_missing": only_missing,
        },
    )
    total_matched = result.rowcount
    db.commit()

    logger.info(
        "match_listings_to_buildings: DONE total_matched=%d threshold_m=%d",
        total_matched,
        threshold_m,
    )
    return total_matched


# ── Combined refresh + match (callable directly or via run wrapper) ───────────
def refresh_and_match(
    db: Session,
    *,
    threshold_m: int = DEFAULT_THRESHOLD_M,
    batch_size: int = DEFAULT_BATCH_SIZE,
    only_missing: bool = True,
) -> CadMatchResult:
    """Refresh the local cad mirror, then geo-match listings. Returns a CadMatchResult."""
    start = time.monotonic()
    res = CadMatchResult(threshold_m=threshold_m)

    res.buildings_loaded = refresh_cad_buildings_local(db)
    res.candidates_total = int(
        db.execute(_CANDIDATES_SQL, {"only_missing": only_missing}).scalar() or 0
    )
    res.listings_matched = match_listings_to_buildings(
        db, threshold_m=threshold_m, batch_size=batch_size, only_missing=only_missing
    )
    res.duration_sec = time.monotonic() - start

    coverage = 100.0 * res.listings_matched / res.candidates_total if res.candidates_total else 0.0
    logger.info(
        "refresh_and_match: DONE buildings=%d candidates=%d matched=%d "
        "coverage=%.1f%% threshold_m=%d duration=%.1fs",
        res.buildings_loaded,
        res.candidates_total,
        res.listings_matched,
        coverage,
        threshold_m,
        res.duration_sec,
    )
    return res


# ── Run lifecycle wrapper (scheduler entrypoint) ─────────────────────────────
def run_cadastral_geo_match(db: Session, *, run_id: int, params: dict) -> CadMatchResult:
    """Run-lifecycle wrapper for the combined refresh+match job (sync, DB-only).

    Launched by the kit scheduler (source='cadastral_geo_match') via
    product_handlers._job_cadastral_geo_match, or manually. Refreshes cad_buildings_local
    from the FDW (one bulk scan) then geo-matches listings.building_cadastral_number to the
    nearest cadastral building within threshold_m.

    Params (default_params jsonb):
        threshold_m:  max nearest-building distance to accept (metres, default 50).
        batch_size:   listings per LATERAL UPDATE chunk (default 5000).
        only_missing: only fill WHERE building_cadastral_number IS NULL (default true).

    Finalises scrape_runs (mark_done / mark_failed) with counters.
    """
    threshold_m = int(params.get("threshold_m", DEFAULT_THRESHOLD_M))
    batch_size = int(params.get("batch_size", DEFAULT_BATCH_SIZE))
    only_missing = bool(params.get("only_missing", True))

    counters: dict[str, int] = {
        "buildings_loaded": 0,
        "candidates_total": 0,
        "listings_matched": 0,
        "threshold_m": threshold_m,
    }

    try:
        runs_mod.update_heartbeat(db, run_id, counters)

        res = refresh_and_match(
            db,
            threshold_m=threshold_m,
            batch_size=batch_size,
            only_missing=only_missing,
        )

        counters = res.to_counters()
        runs_mod.mark_done(db, run_id, counters)
        logger.info(
            "run_cadastral_geo_match: run_id=%d DONE buildings=%d candidates=%d "
            "matched=%d threshold_m=%d duration=%.1fs",
            run_id,
            res.buildings_loaded,
            res.candidates_total,
            res.listings_matched,
            threshold_m,
            res.duration_sec,
        )
        return res
    except Exception as exc:
        logger.exception("run_cadastral_geo_match: run_id=%d FAILED", run_id)
        try:
            db.rollback()
        except Exception:
            pass
        runs_mod.mark_failed(db, run_id, str(exc)[:1000], counters)
        raise