gendesign/tradein-mvp/backend/tests/tasks/test_cadastral_geo_match.py

"""Tests for the geo-nearest cadastral building matcher (#cadastral-geo-match).

Convention mirrors test_listing_source_snapshot / test_asking_to_sold_ratio: the matcher
is SQL-heavy and the gate has no live Postgres, so most assertions are STATIC — we read the
emitted SQL via the text() clauses and inspect.getsource and check:
  - the LATERAL KNN shape (ST_DWithin geography gate + `<->` KNN order + LIMIT 1),
  - the threshold/only_missing params are bound (psycopg-v3 CAST discipline, no :p::type),
  - the refresh does a single bulk FDW scan (TRUNCATE + INSERT ... FROM gendesign_cad_buildings),
  - migration 124 (table + GIST) and 125 (schedule seed) contents.

Plus a fake-db behavioural test driving the chunk-loop + counter logic without Postgres.

An OPTIONAL real-PostGIS behavioural test (test_real_knn_*) asserts the actual
nearest-within-threshold / beyond-threshold→NULL semantics when a PostGIS database is
reachable; it self-SKIPS otherwise so the hermetic gate never depends on it.
"""

from __future__ import annotations

import inspect
import math
import os
import re
from pathlib import Path
from typing import Any

import pytest

# settings needs a DSN at import (same dance as the sibling tests); these are static/fake-db.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

from app.tasks import cadastral_geo_match as cgm

_SQL_DIR = Path(__file__).resolve().parents[2] / "data" / "sql"
_MIGRATION_124 = _SQL_DIR / "124_cad_buildings_local.sql"
_MIGRATION_125 = _SQL_DIR / "125_scrape_schedules_seed_cadastral_geo_match.sql"

_MATCH_SQL = str(cgm._MATCH_SQL.text)
_CANDIDATES_SQL = str(cgm._CANDIDATES_SQL.text)
_REFRESH_SRC = inspect.getsource(cgm.refresh_cad_buildings_local)
_MATCH_SRC = inspect.getsource(cgm.match_listings_to_buildings)


# ── Refresh: single bulk FDW scan ─────────────────────────────────────────────


def test_refresh_truncates_then_bulk_inserts_from_fdw() -> None:
    assert "TRUNCATE cad_buildings_local" in _REFRESH_SRC
    assert "INSERT INTO cad_buildings_local" in _REFRESH_SRC
    assert "FROM gendesign_cad_buildings" in _REFRESH_SRC
    # Builds a Point(4326) geom from lon/lat in the same scan.
    assert "ST_SetSRID(ST_MakePoint(lon, lat), 4326)" in _REFRESH_SRC


def test_refresh_filters_null_coords_and_cad_num() -> None:
    assert "lat IS NOT NULL" in _REFRESH_SRC
    assert "lon IS NOT NULL" in _REFRESH_SRC
    assert "cad_num IS NOT NULL" in _REFRESH_SRC


def test_refresh_is_single_scan_not_per_row() -> None:
    """The FDW must be read once (one INSERT...SELECT), never per listing — the whole point."""
    assert _REFRESH_SRC.count("FROM gendesign_cad_buildings") == 1


# ── Match: LATERAL KNN shape ──────────────────────────────────────────────────


def test_match_uses_lateral_knn_nearest_one() -> None:
    flat = re.sub(r"\s+", " ", _MATCH_SQL)
    assert "JOIN LATERAL" in flat
    # GIST-backed KNN order + single nearest candidate.
    assert "ORDER BY cb.geom <-> l.geom" in flat
    assert "LIMIT 1" in flat


def test_match_gates_with_geometry_dwithin_and_distancesphere_recheck() -> None:
    """Perf-critical: geometry-space ST_DWithin (GIST, degrees) pre-gate + ST_DistanceSphere
    metric recheck — NOT a geography-cast ST_DWithin in the WHERE (that ran 58s+ unfinished)."""
    flat = re.sub(r"\s+", " ", _MATCH_SQL)
    # GIST-indexable degree-space gate (no geography cast on the filtered table).
    assert "ST_DWithin(cb.geom, l.geom, CAST(:deg_gate AS double precision))" in flat
    # The geography cast must NOT appear in the gate (that defeats the index).
    assert "CAST(cb.geom AS geography)" not in flat
    # True metric distance computed only on the single nearest row, gated by threshold_m.
    assert "ST_DistanceSphere(cb.geom, l.geom) AS dist_m" in flat
    assert "m.dist_m <= CAST(:threshold_m AS double precision)" in flat


def test_match_targets_building_cadastral_number() -> None:
    flat = re.sub(r"\s+", " ", _MATCH_SQL)
    assert "UPDATE listings" in flat
    assert "SET building_cadastral_number = matched.cad_num" in flat


def test_match_only_missing_param_is_bound_and_filters() -> None:
    """only_missing=true → only fills WHERE building_cadastral_number IS NULL (idempotent)."""
    flat = re.sub(r"\s+", " ", _MATCH_SQL)
    assert "CAST(:only_missing AS boolean)" in flat
    assert "l.building_cadastral_number IS NULL" in flat
    # geo-present gate so non-geocoded listings never match.
    assert "l.geom IS NOT NULL" in flat


def test_match_is_single_statement_no_offset() -> None:
    # OFFSET-paging over the mutated `building_cadastral_number IS NULL` predicate skips
    # un-processed rows — the matcher must be ONE set-based UPDATE, no OFFSET/LIMIT paging.
    flat = re.sub(r"\s+", " ", _MATCH_SQL)
    assert "OFFSET" not in flat
    assert "ST_DistanceSphere" in flat
    assert "UPDATE listings l" in flat


def test_deg_gate_encloses_threshold() -> None:
    """The degree gate must be a SUPERSET of the metric threshold at EKB latitude.

    A point exactly threshold_m metres away (in the tighter longitude axis) must fall
    INSIDE the gate, else the GIST pre-filter would drop true-positive nearest buildings.
    """
    m_per_deg_lon = cgm._M_PER_DEG_LAT * math.cos(math.radians(cgm._EKB_LAT_DEG))
    for threshold_m in (30, 50, 100):
        deg_gate = cgm._deg_gate_for(float(threshold_m))
        # Longitude-axis metres covered by the gate must exceed the threshold.
        assert deg_gate * m_per_deg_lon >= threshold_m
        # And by the latitude axis too (lat degrees are longer → even more margin).
        assert deg_gate * cgm._M_PER_DEG_LAT >= threshold_m


def test_no_psycopg_v3_colon_colon_cast() -> None:
    """psycopg v3: never :param::type — must use CAST(:param AS type)."""
    # No ':name::type' bound-param cast anywhere in the app SQL (the v3 trap).
    assert not re.search(r":\w+::", _MATCH_SQL)
    assert not re.search(r":\w+::", _CANDIDATES_SQL)
    assert not re.search(r":\w+::", _REFRESH_SRC)


def test_matcher_docstring_marks_approximation_and_threshold() -> None:
    """Spec: the geo-nearest approximation must be explicit + threshold logged."""
    doc = cgm.__doc__ or ""
    assert "GEO-NEAREST" in doc
    match_doc = cgm.match_listings_to_buildings.__doc__ or ""
    assert "GEO-NEAREST" in match_doc
    # threshold logged in match body
    assert "threshold_m=%d" in _MATCH_SRC


# ── Migration content ─────────────────────────────────────────────────────────


def test_migration_124_creates_table_and_gist() -> None:
    sql = _MIGRATION_124.read_text(encoding="utf-8")
    assert "CREATE TABLE IF NOT EXISTS cad_buildings_local" in sql
    assert "cad_num          text PRIMARY KEY" in sql
    assert "geom             geometry(Point, 4326)" in sql
    assert "USING GIST (geom)" in sql
    assert "IF NOT EXISTS cad_buildings_local_geom_idx" in sql
    assert "BEGIN;" in sql and "COMMIT;" in sql
    # The migration must NOT read the FDW (deploy independence) — no FROM/INSERT against it.
    # (The FDW name may appear in comments; only executable references are forbidden.)
    code_lines = [ln for ln in sql.splitlines() if not ln.lstrip().startswith("--")]
    code = "\n".join(code_lines)
    assert "FROM gendesign_cad_buildings" not in code
    assert "INSERT INTO cad_buildings_local" not in code  # populate is the refresh job, not DDL


def test_migration_125_seeds_schedule_not_null_next_run() -> None:
    sql = _MIGRATION_125.read_text(encoding="utf-8")
    assert "INSERT INTO scrape_schedules" in sql
    assert "'cadastral_geo_match'" in sql
    assert "ON CONFLICT (source) DO NOTHING" in sql
    # next_run_at NOT NULL = tomorrow (avoid deploy-time fire).
    assert "CURRENT_DATE + INTERVAL '1 day'" in sql
    assert '"threshold_m": 50' in sql
    assert "BEGIN;" in sql and "COMMIT;" in sql


# ── Fake-db behavioural: chunk loop + counters ───────────────────────────────


class _FakeResult:
    def __init__(self, rowcount: int, scalar: Any = None) -> None:
        self.rowcount = rowcount
        self._scalar = scalar

    def scalar(self) -> Any:
        return self._scalar


class _FakeDB:
    """Minimal Session stand-in: scripts execute() returns queued results in order."""

    def __init__(self, results: list[_FakeResult]) -> None:
        self._results = list(results)
        self.executed: list[str] = []
        self.commits = 0

    def execute(self, clause: Any, params: dict | None = None) -> _FakeResult:
        self.executed.append(str(getattr(clause, "text", clause)))
        return self._results.pop(0)

    def commit(self) -> None:
        self.commits += 1

    def rollback(self) -> None:  # pragma: no cover
        pass


def test_match_runs_single_update_and_counts(monkeypatch: pytest.MonkeyPatch) -> None:
    """Single set-based UPDATE: one execute → rowcount is the match count, one commit."""
    db = _FakeDB([_FakeResult(5)])
    total = cgm.match_listings_to_buildings(db, threshold_m=50, batch_size=10, only_missing=True)
    assert total == 5
    assert db.commits == 1
    assert len(db.executed) == 1  # exactly one UPDATE — no OFFSET paging


def test_run_wrapper_marks_done_with_counters(monkeypatch: pytest.MonkeyPatch) -> None:
    """run_cadastral_geo_match: refresh → candidates → match → mark_done(counters)."""
    marked: dict[str, Any] = {}
    monkeypatch.setattr(cgm.runs_mod, "update_heartbeat", lambda *a, **k: None)
    monkeypatch.setattr(
        cgm.runs_mod,
        "mark_done",
        lambda _db, run_id, counters: marked.update(run_id=run_id, counters=dict(counters)),
    )
    monkeypatch.setattr(cgm.runs_mod, "mark_failed", lambda *a, **k: None)

    # Stub the heavy SQL fns; assert the wrapper plumbs counters correctly.
    monkeypatch.setattr(cgm, "refresh_cad_buildings_local", lambda _db: 36732)
    monkeypatch.setattr(
        cgm,
        "match_listings_to_buildings",
        lambda _db, **k: 12000,
    )

    class _CandDB:
        def execute(self, *a: Any, **k: Any) -> _FakeResult:
            return _FakeResult(0, scalar=40000)

    out = cgm.run_cadastral_geo_match(
        _CandDB(),  # type: ignore[arg-type]
        run_id=7,
        params={"threshold_m": 50, "batch_size": 100, "only_missing": True},
    )
    assert out.buildings_loaded == 36732
    assert out.candidates_total == 40000
    assert out.listings_matched == 12000
    assert out.threshold_m == 50
    assert marked["run_id"] == 7
    assert marked["counters"]["listings_matched"] == 12000
    assert marked["counters"]["threshold_m"] == 50


def test_run_wrapper_marks_failed_on_error(monkeypatch: pytest.MonkeyPatch) -> None:
    failed: dict[str, Any] = {}
    monkeypatch.setattr(cgm.runs_mod, "update_heartbeat", lambda *a, **k: None)
    monkeypatch.setattr(cgm.runs_mod, "mark_done", lambda *a, **k: None)
    monkeypatch.setattr(
        cgm.runs_mod,
        "mark_failed",
        lambda _db, run_id, err, counters: failed.update(run_id=run_id, err=err),
    )

    def _boom(_db: Any) -> int:
        raise RuntimeError("fdw down")

    monkeypatch.setattr(cgm, "refresh_cad_buildings_local", _boom)

    class _DB:
        def rollback(self) -> None:
            pass

    with pytest.raises(RuntimeError):
        cgm.run_cadastral_geo_match(_DB(), run_id=9, params={})  # type: ignore[arg-type]
    assert failed["run_id"] == 9
    assert "fdw down" in failed["err"]


# ── Optional real-PostGIS behavioural test (self-skips without a DB) ──────────


def _live_session() -> Any | None:
    """Return a SQLAlchemy Session if a PostGIS DB is reachable, else None."""
    try:
        from sqlalchemy import create_engine
        from sqlalchemy.orm import sessionmaker

        dsn = os.environ.get("TEST_DATABASE_URL") or os.environ.get("DATABASE_URL", "")
        if not dsn or "localhost:5432/test" in dsn:
            return None
        engine = create_engine(dsn, future=True)
        conn = engine.connect()
        conn.execute(__import__("sqlalchemy").text("SELECT PostGIS_Version()"))
        conn.close()
        return sessionmaker(bind=engine, future=True)()
    except Exception:
        return None


@pytest.mark.skipif(_live_session() is None, reason="no reachable PostGIS test DB")
def test_real_knn_nearest_within_threshold_picked() -> None:
    """With a real PostGIS DB: nearest building within threshold is picked; beyond → NULL."""
    from sqlalchemy import text as _t

    db = _live_session()
    assert db is not None
    try:
        db.execute(
            _t(
                "CREATE TEMP TABLE cad_buildings_local "
                "(LIKE cad_buildings_local INCLUDING ALL) ON COMMIT DROP"
            )
        )
    except Exception:
        # Local table may not exist in this DB → build a minimal temp equivalent.
        db.rollback()
        db.execute(
            _t(
                "CREATE TEMP TABLE cad_buildings_local ("
                "cad_num text PRIMARY KEY, geom geometry(Point,4326)) ON COMMIT DROP"
            )
        )
    # Two buildings: one 10m from the listing, one 500m away.
    db.execute(
        _t(
            "INSERT INTO cad_buildings_local (cad_num, geom) VALUES "
            "('66:01:NEAR', ST_SetSRID(ST_MakePoint(60.6000, 56.8380),4326)),"
            "('66:01:FAR',  ST_SetSRID(ST_MakePoint(60.6100, 56.8380),4326))"
        )
    )
    # Same geometry-gate + ST_DistanceSphere recheck shape as production.
    deg_gate = cgm._deg_gate_for(50.0)
    # listing point ~ at NEAR; nearest within 50m must be 66:01:NEAR.
    row = db.execute(
        _t(
            "SELECT cad_num FROM ("
            "  SELECT cb.cad_num, "
            "         ST_DistanceSphere(cb.geom, ST_SetSRID(ST_MakePoint(60.6001,56.8380),4326)) "
            "             AS dist_m "
            "  FROM cad_buildings_local cb "
            "  WHERE ST_DWithin(cb.geom, ST_SetSRID(ST_MakePoint(60.6001,56.8380),4326), :g) "
            "  ORDER BY cb.geom <-> ST_SetSRID(ST_MakePoint(60.6001,56.8380),4326) LIMIT 1"
            ") m WHERE m.dist_m <= 50"
        ),
        {"g": deg_gate},
    ).fetchone()
    assert row is not None and row[0] == "66:01:NEAR"
    # A point 500m+ from any building → no match within 50m.
    none_row = db.execute(
        _t(
            "SELECT cad_num FROM ("
            "  SELECT cb.cad_num, "
            "         ST_DistanceSphere(cb.geom, ST_SetSRID(ST_MakePoint(60.7000,56.9000),4326)) "
            "             AS dist_m "
            "  FROM cad_buildings_local cb "
            "  WHERE ST_DWithin(cb.geom, ST_SetSRID(ST_MakePoint(60.7000,56.9000),4326), :g) "
            "  ORDER BY cb.geom <-> ST_SetSRID(ST_MakePoint(60.7000,56.9000),4326) LIMIT 1"
            ") m WHERE m.dist_m <= 50"
        ),
        {"g": deg_gate},
    ).fetchone()
    assert none_row is None
    db.rollback()
    db.close()