"""Лоадер реестра зданий ЕКБ из городского геопортала в ekb_geoportal_buildings. Запуск: `python -m app.tasks.ekb_geoportal_ingest` Тайлит bbox ЕКБ (lon 60.45–60.85, lat 56.70–56.95) на мелкую сетку (~250-300 м), чтобы каждый тайл укладывался в server-cap 20000 features. Для каждого тайла запрашивает слой portal_geo_topo_build, группирует part-features по (street_norm, house_norm), считает центроид и UPSERT'ит в ekb_geoportal_buildings. Нормализация ОБЯЗАНА совпадать с geocoder._geoportal_house_match: street_norm = lower(trim(street)) house_norm = lower(house) с удалёнными внутренними пробелами («7 б» → «7б») Configurable через argv/env (для частичного/тестового прогона): --lon-min/--lon-max/--lat-min/--lat-max или env BBOX="lon_min,lat_min,lon_max,lat_max" По умолчанию — полный ЕКБ. """ from __future__ import annotations import argparse import asyncio import logging import os import re from scraper_kit.providers.ekb_geoportal.client import ( EkbGeoportalClient, TopoBuilding, ) from sqlalchemy import text from sqlalchemy.orm import Session from app.core.db import SessionLocal logger = logging.getLogger(__name__) # Полный bbox ЕКБ. EKB_LON_MIN = 60.45 EKB_LON_MAX = 60.85 EKB_LAT_MIN = 56.70 EKB_LAT_MAX = 56.95 # Шаг сетки: ~0.004° lon × ~0.003° lat (≈250-300 м). Центральный тайл 0.005×0.004 # давал 6825 зданий → 0.004×0.003 заведомо < server-cap 20000. TILE_LON_STEP = 0.004 TILE_LAT_STEP = 0.003 # Пауза между запросами — гос-портал, вежливый rate-limit. REQUEST_SLEEP_SEC = 0.1 # Каждые N тайлов — лог прогресса + commit. PROGRESS_EVERY = 50 _INTERNAL_SPACES = re.compile(r"\s+") def normalize_street(street: str) -> str: """street_norm = lower(trim(street)).""" return street.strip().lower() def normalize_house(house: str) -> str: """house_norm = lower(house) с удалёнными внутренними пробелами («7 б» → «7б»).""" return _INTERNAL_SPACES.sub("", house.strip().lower()) def _tile_grid( lon_min: float, lat_min: float, lon_max: float, lat_max: float ) -> list[tuple[float, float, float, float]]: """Сетка тайлов (lon_min, lat_min, lon_max, lat_max), шаг TILE_LON/LAT_STEP.""" tiles: list[tuple[float, float, float, float]] = [] lon = lon_min while lon < lon_max: lon_hi = min(lon + TILE_LON_STEP, lon_max) lat = lat_min while lat < lat_max: lat_hi = min(lat + TILE_LAT_STEP, lat_max) tiles.append((lon, lat, lon_hi, lat_hi)) lat = lat_hi lon = lon_hi return tiles def _group_buildings( buildings: list[TopoBuilding], ) -> dict[tuple[str, str], dict]: """Группирует part-features по (street_norm, house_norm). centroid = среднее центроидов частей; floors = max; purpose/material = первый непустой; source_keys = собранные key. """ grouped: dict[tuple[str, str], dict] = {} for b in buildings: key = (normalize_street(b.street), normalize_house(b.house)) if key not in grouped: grouped[key] = { "street": b.street.strip(), "house": b.house.strip(), "lat_sum": 0.0, "lon_sum": 0.0, "n": 0, "floors": None, "purpose": None, "material": None, "source_keys": [], } g = grouped[key] g["lat_sum"] += b.lat g["lon_sum"] += b.lon g["n"] += 1 if b.floors is not None: g["floors"] = b.floors if g["floors"] is None else max(g["floors"], b.floors) if g["purpose"] is None and b.purpose is not None: g["purpose"] = b.purpose if g["material"] is None and b.material is not None: g["material"] = b.material if b.key is not None and b.key not in g["source_keys"]: g["source_keys"].append(b.key) return grouped def _upsert_building( db: Session, *, street_norm: str, house_norm: str, agg: dict, ) -> None: """UPSERT одного здания в ekb_geoportal_buildings (ON CONFLICT DO UPDATE).""" lat = agg["lat_sum"] / agg["n"] lon = agg["lon_sum"] / agg["n"] db.execute( text(""" INSERT INTO ekb_geoportal_buildings ( street, house, street_norm, house_norm, lat, lon, floors, purpose, material, source_keys, updated_at ) VALUES ( :street, :house, :street_norm, :house_norm, CAST(:lat AS double precision), CAST(:lon AS double precision), CAST(:floors AS integer), :purpose, :material, CAST(:source_keys AS bigint[]), now() ) ON CONFLICT (street_norm, house_norm) DO UPDATE SET street = EXCLUDED.street, house = EXCLUDED.house, lat = EXCLUDED.lat, lon = EXCLUDED.lon, floors = EXCLUDED.floors, purpose = EXCLUDED.purpose, material = EXCLUDED.material, source_keys = EXCLUDED.source_keys, updated_at = now() """), { "street": agg["street"], "house": agg["house"], "street_norm": street_norm, "house_norm": house_norm, "lat": lat, "lon": lon, "floors": agg["floors"], "purpose": agg["purpose"], "material": agg["material"], "source_keys": agg["source_keys"], }, ) async def ingest( db: Session, *, lon_min: float, lat_min: float, lon_max: float, lat_max: float, ) -> int: """Тайлит bbox, тянет здания, UPSERT'ит. Возвращает число upsert'нутых зданий.""" client = EkbGeoportalClient() tiles = _tile_grid(lon_min, lat_min, lon_max, lat_max) total_tiles = len(tiles) logger.info( "geoportal ingest: bbox=(%.4f,%.4f,%.4f,%.4f) → %d тайлов", lon_min, lat_min, lon_max, lat_max, total_tiles, ) upserted = 0 for i, tile in enumerate(tiles, start=1): buildings = await client.fetch_topo_buildings(tile) if buildings: grouped = _group_buildings(buildings) try: with db.begin_nested(): for (street_norm, house_norm), agg in grouped.items(): _upsert_building( db, street_norm=street_norm, house_norm=house_norm, agg=agg, ) upserted += len(grouped) except Exception: logger.warning("geoportal ingest: batch failed for tile=%s", tile, exc_info=True) await asyncio.sleep(REQUEST_SLEEP_SEC) if i % PROGRESS_EVERY == 0 or i == total_tiles: db.commit() logger.info( "geoportal ingest: %d/%d тайлов, upsert'нуто %d зданий, tile=%s", i, total_tiles, upserted, tile, ) db.commit() logger.info("geoportal ingest: готово — upsert'нуто %d зданий", upserted) return upserted def _resolve_bbox(args: argparse.Namespace) -> tuple[float, float, float, float]: """Разрешает bbox: argv → env BBOX → дефолтный ЕКБ.""" env_bbox = os.environ.get("BBOX") if env_bbox: parts = [float(p) for p in env_bbox.split(",")] if len(parts) == 4: return (parts[0], parts[1], parts[2], parts[3]) logger.warning("BBOX env malformed (%r) — игнорирую", env_bbox) return ( args.lon_min if args.lon_min is not None else EKB_LON_MIN, args.lat_min if args.lat_min is not None else EKB_LAT_MIN, args.lon_max if args.lon_max is not None else EKB_LON_MAX, args.lat_max if args.lat_max is not None else EKB_LAT_MAX, ) def main() -> None: logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s", ) parser = argparse.ArgumentParser(description="EKB geoportal building ingest") parser.add_argument("--lon-min", type=float, default=None) parser.add_argument("--lon-max", type=float, default=None) parser.add_argument("--lat-min", type=float, default=None) parser.add_argument("--lat-max", type=float, default=None) args = parser.parse_args() lon_min, lat_min, lon_max, lat_max = _resolve_bbox(args) db = SessionLocal() try: asyncio.run( ingest( db, lon_min=lon_min, lat_min=lat_min, lon_max=lon_max, lat_max=lat_max, ) ) finally: db.close() if __name__ == "__main__": main()