273 lines
9.3 KiB
Python
273 lines
9.3 KiB
Python
"""Лоадер реестра зданий ЕКБ из городского геопортала в ekb_geoportal_buildings.
|
||
|
||
Запуск: `python -m app.tasks.ekb_geoportal_ingest`
|
||
|
||
Тайлит bbox ЕКБ (lon 60.45–60.85, lat 56.70–56.95) на мелкую сетку (~250-300 м),
|
||
чтобы каждый тайл укладывался в server-cap 20000 features. Для каждого тайла
|
||
запрашивает слой portal_geo_topo_build, группирует part-features по
|
||
(street_norm, house_norm), считает центроид и UPSERT'ит в ekb_geoportal_buildings.
|
||
|
||
Нормализация ОБЯЗАНА совпадать с geocoder._geoportal_house_match:
|
||
street_norm = lower(trim(street))
|
||
house_norm = lower(house) с удалёнными внутренними пробелами («7 б» → «7б»)
|
||
|
||
Configurable через argv/env (для частичного/тестового прогона):
|
||
--lon-min/--lon-max/--lat-min/--lat-max или env BBOX="lon_min,lat_min,lon_max,lat_max"
|
||
По умолчанию — полный ЕКБ.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import asyncio
|
||
import logging
|
||
import os
|
||
import re
|
||
|
||
from sqlalchemy import text
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.core.db import SessionLocal
|
||
from app.services.scrapers.ekb_geoportal_client import (
|
||
EkbGeoportalClient,
|
||
TopoBuilding,
|
||
)
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# Полный bbox ЕКБ.
|
||
EKB_LON_MIN = 60.45
|
||
EKB_LON_MAX = 60.85
|
||
EKB_LAT_MIN = 56.70
|
||
EKB_LAT_MAX = 56.95
|
||
|
||
# Шаг сетки: ~0.004° lon × ~0.003° lat (≈250-300 м). Центральный тайл 0.005×0.004
|
||
# давал 6825 зданий → 0.004×0.003 заведомо < server-cap 20000.
|
||
TILE_LON_STEP = 0.004
|
||
TILE_LAT_STEP = 0.003
|
||
|
||
# Пауза между запросами — гос-портал, вежливый rate-limit.
|
||
REQUEST_SLEEP_SEC = 0.1
|
||
|
||
# Каждые N тайлов — лог прогресса + commit.
|
||
PROGRESS_EVERY = 50
|
||
|
||
_INTERNAL_SPACES = re.compile(r"\s+")
|
||
|
||
|
||
def normalize_street(street: str) -> str:
|
||
"""street_norm = lower(trim(street))."""
|
||
return street.strip().lower()
|
||
|
||
|
||
def normalize_house(house: str) -> str:
|
||
"""house_norm = lower(house) с удалёнными внутренними пробелами («7 б» → «7б»)."""
|
||
return _INTERNAL_SPACES.sub("", house.strip().lower())
|
||
|
||
|
||
def _tile_grid(
|
||
lon_min: float, lat_min: float, lon_max: float, lat_max: float
|
||
) -> list[tuple[float, float, float, float]]:
|
||
"""Сетка тайлов (lon_min, lat_min, lon_max, lat_max), шаг TILE_LON/LAT_STEP."""
|
||
tiles: list[tuple[float, float, float, float]] = []
|
||
lon = lon_min
|
||
while lon < lon_max:
|
||
lon_hi = min(lon + TILE_LON_STEP, lon_max)
|
||
lat = lat_min
|
||
while lat < lat_max:
|
||
lat_hi = min(lat + TILE_LAT_STEP, lat_max)
|
||
tiles.append((lon, lat, lon_hi, lat_hi))
|
||
lat = lat_hi
|
||
lon = lon_hi
|
||
return tiles
|
||
|
||
|
||
def _group_buildings(
|
||
buildings: list[TopoBuilding],
|
||
) -> dict[tuple[str, str], dict]:
|
||
"""Группирует part-features по (street_norm, house_norm).
|
||
|
||
centroid = среднее центроидов частей; floors = max; purpose/material = первый
|
||
непустой; source_keys = собранные key.
|
||
"""
|
||
grouped: dict[tuple[str, str], dict] = {}
|
||
for b in buildings:
|
||
key = (normalize_street(b.street), normalize_house(b.house))
|
||
if key not in grouped:
|
||
grouped[key] = {
|
||
"street": b.street.strip(),
|
||
"house": b.house.strip(),
|
||
"lat_sum": 0.0,
|
||
"lon_sum": 0.0,
|
||
"n": 0,
|
||
"floors": None,
|
||
"purpose": None,
|
||
"material": None,
|
||
"source_keys": [],
|
||
}
|
||
g = grouped[key]
|
||
g["lat_sum"] += b.lat
|
||
g["lon_sum"] += b.lon
|
||
g["n"] += 1
|
||
if b.floors is not None:
|
||
g["floors"] = b.floors if g["floors"] is None else max(g["floors"], b.floors)
|
||
if g["purpose"] is None and b.purpose is not None:
|
||
g["purpose"] = b.purpose
|
||
if g["material"] is None and b.material is not None:
|
||
g["material"] = b.material
|
||
if b.key is not None and b.key not in g["source_keys"]:
|
||
g["source_keys"].append(b.key)
|
||
return grouped
|
||
|
||
|
||
def _upsert_building(
|
||
db: Session,
|
||
*,
|
||
street_norm: str,
|
||
house_norm: str,
|
||
agg: dict,
|
||
) -> None:
|
||
"""UPSERT одного здания в ekb_geoportal_buildings (ON CONFLICT DO UPDATE)."""
|
||
lat = agg["lat_sum"] / agg["n"]
|
||
lon = agg["lon_sum"] / agg["n"]
|
||
db.execute(
|
||
text("""
|
||
INSERT INTO ekb_geoportal_buildings (
|
||
street, house, street_norm, house_norm,
|
||
lat, lon, floors, purpose, material, source_keys, updated_at
|
||
) VALUES (
|
||
:street, :house, :street_norm, :house_norm,
|
||
CAST(:lat AS double precision), CAST(:lon AS double precision),
|
||
CAST(:floors AS integer), :purpose, :material,
|
||
CAST(:source_keys AS bigint[]), now()
|
||
)
|
||
ON CONFLICT (street_norm, house_norm) DO UPDATE SET
|
||
street = EXCLUDED.street,
|
||
house = EXCLUDED.house,
|
||
lat = EXCLUDED.lat,
|
||
lon = EXCLUDED.lon,
|
||
floors = EXCLUDED.floors,
|
||
purpose = EXCLUDED.purpose,
|
||
material = EXCLUDED.material,
|
||
source_keys = EXCLUDED.source_keys,
|
||
updated_at = now()
|
||
"""),
|
||
{
|
||
"street": agg["street"],
|
||
"house": agg["house"],
|
||
"street_norm": street_norm,
|
||
"house_norm": house_norm,
|
||
"lat": lat,
|
||
"lon": lon,
|
||
"floors": agg["floors"],
|
||
"purpose": agg["purpose"],
|
||
"material": agg["material"],
|
||
"source_keys": agg["source_keys"],
|
||
},
|
||
)
|
||
|
||
|
||
async def ingest(
|
||
db: Session,
|
||
*,
|
||
lon_min: float,
|
||
lat_min: float,
|
||
lon_max: float,
|
||
lat_max: float,
|
||
) -> int:
|
||
"""Тайлит bbox, тянет здания, UPSERT'ит. Возвращает число upsert'нутых зданий."""
|
||
client = EkbGeoportalClient()
|
||
tiles = _tile_grid(lon_min, lat_min, lon_max, lat_max)
|
||
total_tiles = len(tiles)
|
||
logger.info(
|
||
"geoportal ingest: bbox=(%.4f,%.4f,%.4f,%.4f) → %d тайлов",
|
||
lon_min,
|
||
lat_min,
|
||
lon_max,
|
||
lat_max,
|
||
total_tiles,
|
||
)
|
||
|
||
upserted = 0
|
||
for i, tile in enumerate(tiles, start=1):
|
||
buildings = await client.fetch_topo_buildings(tile)
|
||
if buildings:
|
||
grouped = _group_buildings(buildings)
|
||
try:
|
||
with db.begin_nested():
|
||
for (street_norm, house_norm), agg in grouped.items():
|
||
_upsert_building(
|
||
db,
|
||
street_norm=street_norm,
|
||
house_norm=house_norm,
|
||
agg=agg,
|
||
)
|
||
upserted += len(grouped)
|
||
except Exception:
|
||
logger.warning("geoportal ingest: batch failed for tile=%s", tile, exc_info=True)
|
||
|
||
await asyncio.sleep(REQUEST_SLEEP_SEC)
|
||
|
||
if i % PROGRESS_EVERY == 0 or i == total_tiles:
|
||
db.commit()
|
||
logger.info(
|
||
"geoportal ingest: %d/%d тайлов, upsert'нуто %d зданий, tile=%s",
|
||
i,
|
||
total_tiles,
|
||
upserted,
|
||
tile,
|
||
)
|
||
|
||
db.commit()
|
||
logger.info("geoportal ingest: готово — upsert'нуто %d зданий", upserted)
|
||
return upserted
|
||
|
||
|
||
def _resolve_bbox(args: argparse.Namespace) -> tuple[float, float, float, float]:
|
||
"""Разрешает bbox: argv → env BBOX → дефолтный ЕКБ."""
|
||
env_bbox = os.environ.get("BBOX")
|
||
if env_bbox:
|
||
parts = [float(p) for p in env_bbox.split(",")]
|
||
if len(parts) == 4:
|
||
return (parts[0], parts[1], parts[2], parts[3])
|
||
logger.warning("BBOX env malformed (%r) — игнорирую", env_bbox)
|
||
|
||
return (
|
||
args.lon_min if args.lon_min is not None else EKB_LON_MIN,
|
||
args.lat_min if args.lat_min is not None else EKB_LAT_MIN,
|
||
args.lon_max if args.lon_max is not None else EKB_LON_MAX,
|
||
args.lat_max if args.lat_max is not None else EKB_LAT_MAX,
|
||
)
|
||
|
||
|
||
def main() -> None:
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
||
)
|
||
parser = argparse.ArgumentParser(description="EKB geoportal building ingest")
|
||
parser.add_argument("--lon-min", type=float, default=None)
|
||
parser.add_argument("--lon-max", type=float, default=None)
|
||
parser.add_argument("--lat-min", type=float, default=None)
|
||
parser.add_argument("--lat-max", type=float, default=None)
|
||
args = parser.parse_args()
|
||
|
||
lon_min, lat_min, lon_max, lat_max = _resolve_bbox(args)
|
||
|
||
db = SessionLocal()
|
||
try:
|
||
asyncio.run(
|
||
ingest(
|
||
db,
|
||
lon_min=lon_min,
|
||
lat_min=lat_min,
|
||
lon_max=lon_max,
|
||
lat_max=lat_max,
|
||
)
|
||
)
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|