gendesign/tradein-mvp/backend/app/tasks/ekb_geoportal_ingest.py

"""Лоадер реестра зданий ЕКБ из городского геопортала в ekb_geoportal_buildings.

Запуск: `python -m app.tasks.ekb_geoportal_ingest`

Тайлит bbox ЕКБ (lon 60.45–60.85, lat 56.70–56.95) на мелкую сетку (~250-300 м),
чтобы каждый тайл укладывался в server-cap 20000 features. Для каждого тайла
запрашивает слой portal_geo_topo_build, группирует part-features по
(street_norm, house_norm), считает центроид и UPSERT'ит в ekb_geoportal_buildings.

Нормализация ОБЯЗАНА совпадать с geocoder._geoportal_house_match:
  street_norm = lower(trim(street))
  house_norm  = lower(house) с удалёнными внутренними пробелами («7 б» → «7б»)

Configurable через argv/env (для частичного/тестового прогона):
  --lon-min/--lon-max/--lat-min/--lat-max  или env BBOX="lon_min,lat_min,lon_max,lat_max"
По умолчанию — полный ЕКБ.
"""

from __future__ import annotations

import argparse
import asyncio
import logging
import os
import re

from scraper_kit.providers.ekb_geoportal.client import (
    EkbGeoportalClient,
    TopoBuilding,
)
from sqlalchemy import text
from sqlalchemy.orm import Session

from app.core.db import SessionLocal

logger = logging.getLogger(__name__)

# Полный bbox ЕКБ.
EKB_LON_MIN = 60.45
EKB_LON_MAX = 60.85
EKB_LAT_MIN = 56.70
EKB_LAT_MAX = 56.95

# Шаг сетки: ~0.004° lon × ~0.003° lat (≈250-300 м). Центральный тайл 0.005×0.004
# давал 6825 зданий → 0.004×0.003 заведомо < server-cap 20000.
TILE_LON_STEP = 0.004
TILE_LAT_STEP = 0.003

# Пауза между запросами — гос-портал, вежливый rate-limit.
REQUEST_SLEEP_SEC = 0.1

# Каждые N тайлов — лог прогресса + commit.
PROGRESS_EVERY = 50

_INTERNAL_SPACES = re.compile(r"\s+")


def normalize_street(street: str) -> str:
    """street_norm = lower(trim(street))."""
    return street.strip().lower()


def normalize_house(house: str) -> str:
    """house_norm = lower(house) с удалёнными внутренними пробелами («7 б» → «7б»)."""
    return _INTERNAL_SPACES.sub("", house.strip().lower())


def _tile_grid(
    lon_min: float, lat_min: float, lon_max: float, lat_max: float
) -> list[tuple[float, float, float, float]]:
    """Сетка тайлов (lon_min, lat_min, lon_max, lat_max), шаг TILE_LON/LAT_STEP."""
    tiles: list[tuple[float, float, float, float]] = []
    lon = lon_min
    while lon < lon_max:
        lon_hi = min(lon + TILE_LON_STEP, lon_max)
        lat = lat_min
        while lat < lat_max:
            lat_hi = min(lat + TILE_LAT_STEP, lat_max)
            tiles.append((lon, lat, lon_hi, lat_hi))
            lat = lat_hi
        lon = lon_hi
    return tiles


def _group_buildings(
    buildings: list[TopoBuilding],
) -> dict[tuple[str, str], dict]:
    """Группирует part-features по (street_norm, house_norm).

    centroid = среднее центроидов частей; floors = max; purpose/material = первый
    непустой; source_keys = собранные key.
    """
    grouped: dict[tuple[str, str], dict] = {}
    for b in buildings:
        key = (normalize_street(b.street), normalize_house(b.house))
        if key not in grouped:
            grouped[key] = {
                "street": b.street.strip(),
                "house": b.house.strip(),
                "lat_sum": 0.0,
                "lon_sum": 0.0,
                "n": 0,
                "floors": None,
                "purpose": None,
                "material": None,
                "source_keys": [],
            }
        g = grouped[key]
        g["lat_sum"] += b.lat
        g["lon_sum"] += b.lon
        g["n"] += 1
        if b.floors is not None:
            g["floors"] = b.floors if g["floors"] is None else max(g["floors"], b.floors)
        if g["purpose"] is None and b.purpose is not None:
            g["purpose"] = b.purpose
        if g["material"] is None and b.material is not None:
            g["material"] = b.material
        if b.key is not None and b.key not in g["source_keys"]:
            g["source_keys"].append(b.key)
    return grouped


def _upsert_building(
    db: Session,
    *,
    street_norm: str,
    house_norm: str,
    agg: dict,
) -> None:
    """UPSERT одного здания в ekb_geoportal_buildings (ON CONFLICT DO UPDATE)."""
    lat = agg["lat_sum"] / agg["n"]
    lon = agg["lon_sum"] / agg["n"]
    db.execute(
        text("""
            INSERT INTO ekb_geoportal_buildings (
                street, house, street_norm, house_norm,
                lat, lon, floors, purpose, material, source_keys, updated_at
            ) VALUES (
                :street, :house, :street_norm, :house_norm,
                CAST(:lat AS double precision), CAST(:lon AS double precision),
                CAST(:floors AS integer), :purpose, :material,
                CAST(:source_keys AS bigint[]), now()
            )
            ON CONFLICT (street_norm, house_norm) DO UPDATE SET
                street      = EXCLUDED.street,
                house       = EXCLUDED.house,
                lat         = EXCLUDED.lat,
                lon         = EXCLUDED.lon,
                floors      = EXCLUDED.floors,
                purpose     = EXCLUDED.purpose,
                material    = EXCLUDED.material,
                source_keys = EXCLUDED.source_keys,
                updated_at  = now()
        """),
        {
            "street": agg["street"],
            "house": agg["house"],
            "street_norm": street_norm,
            "house_norm": house_norm,
            "lat": lat,
            "lon": lon,
            "floors": agg["floors"],
            "purpose": agg["purpose"],
            "material": agg["material"],
            "source_keys": agg["source_keys"],
        },
    )


async def ingest(
    db: Session,
    *,
    lon_min: float,
    lat_min: float,
    lon_max: float,
    lat_max: float,
) -> int:
    """Тайлит bbox, тянет здания, UPSERT'ит. Возвращает число upsert'нутых зданий."""
    client = EkbGeoportalClient()
    tiles = _tile_grid(lon_min, lat_min, lon_max, lat_max)
    total_tiles = len(tiles)
    logger.info(
        "geoportal ingest: bbox=(%.4f,%.4f,%.4f,%.4f) → %d тайлов",
        lon_min,
        lat_min,
        lon_max,
        lat_max,
        total_tiles,
    )

    upserted = 0
    for i, tile in enumerate(tiles, start=1):
        buildings = await client.fetch_topo_buildings(tile)
        if buildings:
            grouped = _group_buildings(buildings)
            try:
                with db.begin_nested():
                    for (street_norm, house_norm), agg in grouped.items():
                        _upsert_building(
                            db,
                            street_norm=street_norm,
                            house_norm=house_norm,
                            agg=agg,
                        )
                upserted += len(grouped)
            except Exception:
                logger.warning("geoportal ingest: batch failed for tile=%s", tile, exc_info=True)

        await asyncio.sleep(REQUEST_SLEEP_SEC)

        if i % PROGRESS_EVERY == 0 or i == total_tiles:
            db.commit()
            logger.info(
                "geoportal ingest: %d/%d тайлов, upsert'нуто %d зданий, tile=%s",
                i,
                total_tiles,
                upserted,
                tile,
            )

    db.commit()
    logger.info("geoportal ingest: готово — upsert'нуто %d зданий", upserted)
    return upserted


def _resolve_bbox(args: argparse.Namespace) -> tuple[float, float, float, float]:
    """Разрешает bbox: argv → env BBOX → дефолтный ЕКБ."""
    env_bbox = os.environ.get("BBOX")
    if env_bbox:
        parts = [float(p) for p in env_bbox.split(",")]
        if len(parts) == 4:
            return (parts[0], parts[1], parts[2], parts[3])
        logger.warning("BBOX env malformed (%r) — игнорирую", env_bbox)

    return (
        args.lon_min if args.lon_min is not None else EKB_LON_MIN,
        args.lat_min if args.lat_min is not None else EKB_LAT_MIN,
        args.lon_max if args.lon_max is not None else EKB_LON_MAX,
        args.lat_max if args.lat_max is not None else EKB_LAT_MAX,
    )


def main() -> None:
    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
    )
    parser = argparse.ArgumentParser(description="EKB geoportal building ingest")
    parser.add_argument("--lon-min", type=float, default=None)
    parser.add_argument("--lon-max", type=float, default=None)
    parser.add_argument("--lat-min", type=float, default=None)
    parser.add_argument("--lat-max", type=float, default=None)
    args = parser.parse_args()

    lon_min, lat_min, lon_max, lat_max = _resolve_bbox(args)

    db = SessionLocal()
    try:
        asyncio.run(
            ingest(
                db,
                lon_min=lon_min,
                lat_min=lat_min,
                lon_max=lon_max,
                lat_max=lat_max,
            )
        )
    finally:
        db.close()


if __name__ == "__main__":
    main()