"""Tests for the geo-nearest cadastral building matcher (#cadastral-geo-match). Convention mirrors test_listing_source_snapshot / test_asking_to_sold_ratio: the matcher is SQL-heavy and the gate has no live Postgres, so most assertions are STATIC — we read the emitted SQL via the text() clauses and inspect.getsource and check: - the LATERAL KNN shape (ST_DWithin geography gate + `<->` KNN order + LIMIT 1), - the threshold/only_missing params are bound (psycopg-v3 CAST discipline, no :p::type), - the refresh does a single bulk FDW scan (TRUNCATE + INSERT ... FROM gendesign_cad_buildings), - migration 124 (table + GIST) and 125 (schedule seed) contents. Plus a fake-db behavioural test driving the chunk-loop + counter logic without Postgres. An OPTIONAL real-PostGIS behavioural test (test_real_knn_*) asserts the actual nearest-within-threshold / beyond-threshold→NULL semantics when a PostGIS database is reachable; it self-SKIPS otherwise so the hermetic gate never depends on it. """ from __future__ import annotations import inspect import math import os import re from pathlib import Path from typing import Any import pytest # settings needs a DSN at import (same dance as the sibling tests); these are static/fake-db. os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") from app.tasks import cadastral_geo_match as cgm _SQL_DIR = Path(__file__).resolve().parents[2] / "data" / "sql" _MIGRATION_124 = _SQL_DIR / "124_cad_buildings_local.sql" _MIGRATION_125 = _SQL_DIR / "125_scrape_schedules_seed_cadastral_geo_match.sql" _MATCH_SQL = str(cgm._MATCH_SQL.text) _CANDIDATES_SQL = str(cgm._CANDIDATES_SQL.text) _REFRESH_SRC = inspect.getsource(cgm.refresh_cad_buildings_local) _MATCH_SRC = inspect.getsource(cgm.match_listings_to_buildings) # ── Refresh: single bulk FDW scan ───────────────────────────────────────────── def test_refresh_truncates_then_bulk_inserts_from_fdw() -> None: assert "TRUNCATE cad_buildings_local" in _REFRESH_SRC assert "INSERT INTO cad_buildings_local" in _REFRESH_SRC assert "FROM gendesign_cad_buildings" in _REFRESH_SRC # Builds a Point(4326) geom from lon/lat in the same scan. assert "ST_SetSRID(ST_MakePoint(lon, lat), 4326)" in _REFRESH_SRC def test_refresh_filters_null_coords_and_cad_num() -> None: assert "lat IS NOT NULL" in _REFRESH_SRC assert "lon IS NOT NULL" in _REFRESH_SRC assert "cad_num IS NOT NULL" in _REFRESH_SRC def test_refresh_is_single_scan_not_per_row() -> None: """The FDW must be read once (one INSERT...SELECT), never per listing — the whole point.""" assert _REFRESH_SRC.count("FROM gendesign_cad_buildings") == 1 # ── Match: LATERAL KNN shape ────────────────────────────────────────────────── def test_match_uses_lateral_knn_nearest_one() -> None: flat = re.sub(r"\s+", " ", _MATCH_SQL) assert "JOIN LATERAL" in flat # GIST-backed KNN order + single nearest candidate. assert "ORDER BY cb.geom <-> l.geom" in flat assert "LIMIT 1" in flat def test_match_gates_with_geometry_dwithin_and_distancesphere_recheck() -> None: """Perf-critical: geometry-space ST_DWithin (GIST, degrees) pre-gate + ST_DistanceSphere metric recheck — NOT a geography-cast ST_DWithin in the WHERE (that ran 58s+ unfinished).""" flat = re.sub(r"\s+", " ", _MATCH_SQL) # GIST-indexable degree-space gate (no geography cast on the filtered table). assert "ST_DWithin(cb.geom, l.geom, CAST(:deg_gate AS double precision))" in flat # The geography cast must NOT appear in the gate (that defeats the index). assert "CAST(cb.geom AS geography)" not in flat # True metric distance computed only on the single nearest row, gated by threshold_m. assert "ST_DistanceSphere(cb.geom, l.geom) AS dist_m" in flat assert "m.dist_m <= CAST(:threshold_m AS double precision)" in flat def test_match_targets_building_cadastral_number() -> None: flat = re.sub(r"\s+", " ", _MATCH_SQL) assert "UPDATE listings" in flat assert "SET building_cadastral_number = matched.cad_num" in flat def test_match_only_missing_param_is_bound_and_filters() -> None: """only_missing=true → only fills WHERE building_cadastral_number IS NULL (idempotent).""" flat = re.sub(r"\s+", " ", _MATCH_SQL) assert "CAST(:only_missing AS boolean)" in flat assert "l.building_cadastral_number IS NULL" in flat # geo-present gate so non-geocoded listings never match. assert "l.geom IS NOT NULL" in flat def test_match_is_single_statement_no_offset() -> None: # OFFSET-paging over the mutated `building_cadastral_number IS NULL` predicate skips # un-processed rows — the matcher must be ONE set-based UPDATE, no OFFSET/LIMIT paging. flat = re.sub(r"\s+", " ", _MATCH_SQL) assert "OFFSET" not in flat assert "ST_DistanceSphere" in flat assert "UPDATE listings l" in flat def test_deg_gate_encloses_threshold() -> None: """The degree gate must be a SUPERSET of the metric threshold at EKB latitude. A point exactly threshold_m metres away (in the tighter longitude axis) must fall INSIDE the gate, else the GIST pre-filter would drop true-positive nearest buildings. """ m_per_deg_lon = cgm._M_PER_DEG_LAT * math.cos(math.radians(cgm._EKB_LAT_DEG)) for threshold_m in (30, 50, 100): deg_gate = cgm._deg_gate_for(float(threshold_m)) # Longitude-axis metres covered by the gate must exceed the threshold. assert deg_gate * m_per_deg_lon >= threshold_m # And by the latitude axis too (lat degrees are longer → even more margin). assert deg_gate * cgm._M_PER_DEG_LAT >= threshold_m def test_no_psycopg_v3_colon_colon_cast() -> None: """psycopg v3: never :param::type — must use CAST(:param AS type).""" # No ':name::type' bound-param cast anywhere in the app SQL (the v3 trap). assert not re.search(r":\w+::", _MATCH_SQL) assert not re.search(r":\w+::", _CANDIDATES_SQL) assert not re.search(r":\w+::", _REFRESH_SRC) def test_matcher_docstring_marks_approximation_and_threshold() -> None: """Spec: the geo-nearest approximation must be explicit + threshold logged.""" doc = cgm.__doc__ or "" assert "GEO-NEAREST" in doc match_doc = cgm.match_listings_to_buildings.__doc__ or "" assert "GEO-NEAREST" in match_doc # threshold logged in match body assert "threshold_m=%d" in _MATCH_SRC # ── Migration content ───────────────────────────────────────────────────────── def test_migration_124_creates_table_and_gist() -> None: sql = _MIGRATION_124.read_text(encoding="utf-8") assert "CREATE TABLE IF NOT EXISTS cad_buildings_local" in sql assert "cad_num text PRIMARY KEY" in sql assert "geom geometry(Point, 4326)" in sql assert "USING GIST (geom)" in sql assert "IF NOT EXISTS cad_buildings_local_geom_idx" in sql assert "BEGIN;" in sql and "COMMIT;" in sql # The migration must NOT read the FDW (deploy independence) — no FROM/INSERT against it. # (The FDW name may appear in comments; only executable references are forbidden.) code_lines = [ln for ln in sql.splitlines() if not ln.lstrip().startswith("--")] code = "\n".join(code_lines) assert "FROM gendesign_cad_buildings" not in code assert "INSERT INTO cad_buildings_local" not in code # populate is the refresh job, not DDL def test_migration_125_seeds_schedule_not_null_next_run() -> None: sql = _MIGRATION_125.read_text(encoding="utf-8") assert "INSERT INTO scrape_schedules" in sql assert "'cadastral_geo_match'" in sql assert "ON CONFLICT (source) DO NOTHING" in sql # next_run_at NOT NULL = tomorrow (avoid deploy-time fire). assert "CURRENT_DATE + INTERVAL '1 day'" in sql assert '"threshold_m": 50' in sql assert "BEGIN;" in sql and "COMMIT;" in sql # ── Fake-db behavioural: chunk loop + counters ─────────────────────────────── class _FakeResult: def __init__(self, rowcount: int, scalar: Any = None) -> None: self.rowcount = rowcount self._scalar = scalar def scalar(self) -> Any: return self._scalar class _FakeDB: """Minimal Session stand-in: scripts execute() returns queued results in order.""" def __init__(self, results: list[_FakeResult]) -> None: self._results = list(results) self.executed: list[str] = [] self.commits = 0 def execute(self, clause: Any, params: dict | None = None) -> _FakeResult: self.executed.append(str(getattr(clause, "text", clause))) return self._results.pop(0) def commit(self) -> None: self.commits += 1 def rollback(self) -> None: # pragma: no cover pass def test_match_runs_single_update_and_counts(monkeypatch: pytest.MonkeyPatch) -> None: """Single set-based UPDATE: one execute → rowcount is the match count, one commit.""" db = _FakeDB([_FakeResult(5)]) total = cgm.match_listings_to_buildings(db, threshold_m=50, batch_size=10, only_missing=True) assert total == 5 assert db.commits == 1 assert len(db.executed) == 1 # exactly one UPDATE — no OFFSET paging def test_run_wrapper_marks_done_with_counters(monkeypatch: pytest.MonkeyPatch) -> None: """run_cadastral_geo_match: refresh → candidates → match → mark_done(counters).""" marked: dict[str, Any] = {} monkeypatch.setattr(cgm.runs_mod, "update_heartbeat", lambda *a, **k: None) monkeypatch.setattr( cgm.runs_mod, "mark_done", lambda _db, run_id, counters: marked.update(run_id=run_id, counters=dict(counters)), ) monkeypatch.setattr(cgm.runs_mod, "mark_failed", lambda *a, **k: None) # Stub the heavy SQL fns; assert the wrapper plumbs counters correctly. monkeypatch.setattr(cgm, "refresh_cad_buildings_local", lambda _db: 36732) monkeypatch.setattr( cgm, "match_listings_to_buildings", lambda _db, **k: 12000, ) class _CandDB: def execute(self, *a: Any, **k: Any) -> _FakeResult: return _FakeResult(0, scalar=40000) out = cgm.run_cadastral_geo_match( _CandDB(), # type: ignore[arg-type] run_id=7, params={"threshold_m": 50, "batch_size": 100, "only_missing": True}, ) assert out.buildings_loaded == 36732 assert out.candidates_total == 40000 assert out.listings_matched == 12000 assert out.threshold_m == 50 assert marked["run_id"] == 7 assert marked["counters"]["listings_matched"] == 12000 assert marked["counters"]["threshold_m"] == 50 def test_run_wrapper_marks_failed_on_error(monkeypatch: pytest.MonkeyPatch) -> None: failed: dict[str, Any] = {} monkeypatch.setattr(cgm.runs_mod, "update_heartbeat", lambda *a, **k: None) monkeypatch.setattr(cgm.runs_mod, "mark_done", lambda *a, **k: None) monkeypatch.setattr( cgm.runs_mod, "mark_failed", lambda _db, run_id, err, counters: failed.update(run_id=run_id, err=err), ) def _boom(_db: Any) -> int: raise RuntimeError("fdw down") monkeypatch.setattr(cgm, "refresh_cad_buildings_local", _boom) class _DB: def rollback(self) -> None: pass with pytest.raises(RuntimeError): cgm.run_cadastral_geo_match(_DB(), run_id=9, params={}) # type: ignore[arg-type] assert failed["run_id"] == 9 assert "fdw down" in failed["err"] # ── Optional real-PostGIS behavioural test (self-skips without a DB) ────────── def _live_session() -> Any | None: """Return a SQLAlchemy Session if a PostGIS DB is reachable, else None.""" try: from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker dsn = os.environ.get("TEST_DATABASE_URL") or os.environ.get("DATABASE_URL", "") if not dsn or "localhost:5432/test" in dsn: return None engine = create_engine(dsn, future=True) conn = engine.connect() conn.execute(__import__("sqlalchemy").text("SELECT PostGIS_Version()")) conn.close() return sessionmaker(bind=engine, future=True)() except Exception: return None @pytest.mark.skipif(_live_session() is None, reason="no reachable PostGIS test DB") def test_real_knn_nearest_within_threshold_picked() -> None: """With a real PostGIS DB: nearest building within threshold is picked; beyond → NULL.""" from sqlalchemy import text as _t db = _live_session() assert db is not None try: db.execute( _t( "CREATE TEMP TABLE cad_buildings_local " "(LIKE cad_buildings_local INCLUDING ALL) ON COMMIT DROP" ) ) except Exception: # Local table may not exist in this DB → build a minimal temp equivalent. db.rollback() db.execute( _t( "CREATE TEMP TABLE cad_buildings_local (" "cad_num text PRIMARY KEY, geom geometry(Point,4326)) ON COMMIT DROP" ) ) # Two buildings: one 10m from the listing, one 500m away. db.execute( _t( "INSERT INTO cad_buildings_local (cad_num, geom) VALUES " "('66:01:NEAR', ST_SetSRID(ST_MakePoint(60.6000, 56.8380),4326))," "('66:01:FAR', ST_SetSRID(ST_MakePoint(60.6100, 56.8380),4326))" ) ) # Same geometry-gate + ST_DistanceSphere recheck shape as production. deg_gate = cgm._deg_gate_for(50.0) # listing point ~ at NEAR; nearest within 50m must be 66:01:NEAR. row = db.execute( _t( "SELECT cad_num FROM (" " SELECT cb.cad_num, " " ST_DistanceSphere(cb.geom, ST_SetSRID(ST_MakePoint(60.6001,56.8380),4326)) " " AS dist_m " " FROM cad_buildings_local cb " " WHERE ST_DWithin(cb.geom, ST_SetSRID(ST_MakePoint(60.6001,56.8380),4326), :g) " " ORDER BY cb.geom <-> ST_SetSRID(ST_MakePoint(60.6001,56.8380),4326) LIMIT 1" ") m WHERE m.dist_m <= 50" ), {"g": deg_gate}, ).fetchone() assert row is not None and row[0] == "66:01:NEAR" # A point 500m+ from any building → no match within 50m. none_row = db.execute( _t( "SELECT cad_num FROM (" " SELECT cb.cad_num, " " ST_DistanceSphere(cb.geom, ST_SetSRID(ST_MakePoint(60.7000,56.9000),4326)) " " AS dist_m " " FROM cad_buildings_local cb " " WHERE ST_DWithin(cb.geom, ST_SetSRID(ST_MakePoint(60.7000,56.9000),4326), :g) " " ORDER BY cb.geom <-> ST_SetSRID(ST_MakePoint(60.7000,56.9000),4326) LIMIT 1" ") m WHERE m.dist_m <= 50" ), {"g": deg_gate}, ).fetchone() assert none_row is None db.rollback() db.close()