gendesign/tradein-mvp/backend/app/tasks/osm_poi_ekb_refresh.py

"""OSM POI local-mirror refresh (#2045 BE-3, LocationDrawer location-index).

Populates `osm_poi_ekb_local` (empty at deploy, migration 169) via a single bulk scan of the
`gendesign_osm_poi_ekb` FDW foreign table (migration 168 — live view of gendesign's
`osm_poi_ekb` table, Site Finder's OSM POI registry, GRANTed to `tradein_fdw_reader` in PR-A
of #2045, already merged + deployed on gendesign).

WHY a local mirror (perf fact measured for the analogous gendesign_cad_buildings FDW — see
`app/tasks/cadastral_geo_match.py`): a per-request FDW nearest-POI query pays a per-row FDW
round-trip (~1.16s/row without a geom index on the remote table) — UNUSABLE for a synchronous
endpoint (`GET /api/v1/trade-in/location-index`). We materialize the FDW once (single bulk
scan) into `osm_poi_ekb_local` with a real Point geom + GIST index, then
`app/services/location_index.py` runs fast LOCAL ST_DWithin/ST_Distance queries per estimate
(nearby-POI qualitative list only — the numeric index itself comes from `listings`, not POI;
see that module's docstring for the location-coef → location-index rewrite history).

Scheduler source='osm_poi_ekb_refresh' (daily — OSM POI data changes rarely). Pure internal
DB op — one FDW read + local TRUNCATE+INSERT, no HTTP/anti-bot.

psycopg v3: app SQL uses CAST(:x AS type), never :x::type.
"""

from __future__ import annotations

import logging
import time
from dataclasses import dataclass, field

from sqlalchemy import text
from sqlalchemy.orm import Session

from app.services import scrape_runs as runs_mod

logger = logging.getLogger(__name__)


def refresh_osm_poi_ekb_local(db: Session) -> int:
    """TRUNCATE osm_poi_ekb_local; bulk INSERT from gendesign_osm_poi_ekb FDW.

    Single FDW scan (no per-row round-trips). Builds a Point(4326) geom from lon/lat.
    Idempotent: TRUNCATE+INSERT in one transaction — the table is fully replaced atomically.

    Returns the number of rows loaded.
    """
    db.execute(text("TRUNCATE osm_poi_ekb_local"))
    db.execute(
        text(
            """
            INSERT INTO osm_poi_ekb_local (category, name, lat, lon, geom)
            SELECT
                category, name, lat, lon,
                ST_SetSRID(ST_MakePoint(lon, lat), 4326)
            FROM gendesign_osm_poi_ekb
            WHERE lat IS NOT NULL
              AND lon IS NOT NULL
            """
        )
    )
    count = db.execute(text("SELECT count(*) FROM osm_poi_ekb_local")).scalar() or 0
    db.commit()
    logger.info("refresh_osm_poi_ekb_local: loaded %d rows from FDW", count)
    return int(count)


@dataclass
class OsmPoiRefreshResult:
    poi_loaded: int = 0  # rows in osm_poi_ekb_local after refresh
    duration_sec: float = field(default=0.0)

    def to_counters(self) -> dict[str, int]:
        return {
            "poi_loaded": self.poi_loaded,
            "duration_sec": int(self.duration_sec),
        }


def run_osm_poi_ekb_refresh(db: Session, *, run_id: int, params: dict) -> OsmPoiRefreshResult:
    """Run-lifecycle wrapper for the OSM POI local-mirror refresh (scheduler entrypoint).

    Launched by the kit scheduler (source='osm_poi_ekb_refresh') via
    product_handlers._job_osm_poi_ekb_refresh, or manually. No params consumed — single
    unconditional bulk refresh (`params` accepted for scheduler-signature compatibility).

    Finalises scrape_runs (mark_done / mark_failed) with counters.
    """
    counters: dict[str, int] = {"poi_loaded": 0}
    start = time.monotonic()

    try:
        runs_mod.update_heartbeat(db, run_id, counters)

        loaded = refresh_osm_poi_ekb_local(db)
        result = OsmPoiRefreshResult(poi_loaded=loaded, duration_sec=time.monotonic() - start)

        counters = result.to_counters()
        runs_mod.mark_done(db, run_id, counters)
        logger.info(
            "run_osm_poi_ekb_refresh: run_id=%d DONE poi_loaded=%d duration=%.1fs",
            run_id,
            result.poi_loaded,
            result.duration_sec,
        )
        return result
    except Exception as exc:
        logger.exception("run_osm_poi_ekb_refresh: run_id=%d FAILED", run_id)
        try:
            db.rollback()
        except Exception:
            pass
        runs_mod.mark_failed(db, run_id, str(exc)[:1000], counters)
        raise