From 3753079deeea23e39f17daaa9d9fa4506e424286 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Fri, 26 Jun 2026 20:01:00 +0000 Subject: [PATCH] feat(site-finder): OSM engineering-networks loader + endpoint (#1746) (#1930) --- backend/app/api/v1/parcels.py | 36 ++ backend/app/schemas/parcel.py | 28 ++ .../utility_infrastructure_loader.py | 390 ++++++++++++++++++ backend/app/workers/beat_schedule.py | 9 + backend/app/workers/celery_app.py | 1 + .../tasks/utility_infrastructure_sync.py | 59 +++ .../v1/test_parcel_utility_infrastructure.py | 136 ++++++ .../test_utility_infrastructure_loader.py | 284 +++++++++++++ .../170_osm_utility_infrastructure_ekb.sql | 53 +++ frontend/src/lib/api-types.ts | 98 +++++ 10 files changed, 1094 insertions(+) create mode 100644 backend/app/services/site_finder/utility_infrastructure_loader.py create mode 100644 backend/app/workers/tasks/utility_infrastructure_sync.py create mode 100644 backend/tests/api/v1/test_parcel_utility_infrastructure.py create mode 100644 backend/tests/test_utility_infrastructure_loader.py create mode 100644 data/sql/170_osm_utility_infrastructure_ekb.sql diff --git a/backend/app/api/v1/parcels.py b/backend/app/api/v1/parcels.py index dcade81f..f222b3a0 100644 --- a/backend/app/api/v1/parcels.py +++ b/backend/app/api/v1/parcels.py @@ -35,6 +35,7 @@ from app.schemas.parcel import ( ParcelSearchResponse, RedLine, RiskZone, + UtilityInfrastructureResponse, ) from app.services.analysis_runs.repository import ( ANALYZE_SCHEMA_VERSION, @@ -82,6 +83,9 @@ from app.services.site_finder.quarter_dump_lookup import ( ) from app.services.site_finder.riasurt_lookup import parcel_riasurt_gate from app.services.site_finder.saturation import compute_district_saturation +from app.services.site_finder.utility_infrastructure_loader import ( + get_nearby_utility_infrastructure, +) from app.services.site_finder.velocity import compute_velocity from app.services.site_finder.weight_profiles import ( _SYSTEM_POI_WEIGHTS as _POI_WEIGHTS, @@ -3435,6 +3439,38 @@ def get_parcel_connection_points( ) +@router.get( + "/{cad_num}/utility-infrastructure", + response_model=UtilityInfrastructureResponse, + summary="Инженерные сети из OSM вблизи участка (#1746)", +) +def get_parcel_utility_infrastructure( + cad_num: str, + db: Annotated[Session, Depends(get_db)], + radius_m: Annotated[int, Query(ge=50, le=2000)] = 500, +) -> UtilityInfrastructureResponse: + """Инженерные сети (ЛЭП/подстанции/трубопроводы/ЦТП/вышки) из OSM вблизи участка. + + No-B2B open-data источник: osm_utility_infrastructure_ekb (наполняется weekly-sync + из Overpass). Возвращает фичи в радиусе от центроида участка (ST_DWithin) + + summary с ближайшими расстояниями по видам сети. + + Query params: + radius_m: радиус поиска, 50–2000 м (default 500). + + Если участок не найден в БД (нет geom) → 404. + """ + try: + data = get_nearby_utility_infrastructure(db, cad_num, radius_m) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + return UtilityInfrastructureResponse( + features=data["features"], + summary=data["summary"], + ) + + _ORS_BASE = "https://api.openrouteservice.org/v2/isochrones" _ORS_VALID_MODES = frozenset({"foot-walking", "cycling-regular", "driving-car"}) diff --git a/backend/app/schemas/parcel.py b/backend/app/schemas/parcel.py index a0394991..72ace881 100644 --- a/backend/app/schemas/parcel.py +++ b/backend/app/schemas/parcel.py @@ -68,6 +68,34 @@ class ConnectionPointsResponse(BaseModel): dump_fetched_at: str | None +# ── OSM engineering-networks schemas (#1746) ───────────────────────────────── +# No-B2B open-data слой инженерных сетей из OSM (Overpass): power/water/gas/heat/ +# communication/sewage. Источник — osm_utility_infrastructure_ekb. + + +class UtilityInfrastructureFeature(BaseModel): + osm_id: int + osm_type: str + # 'power' | 'water' | 'gas' | 'heat' | 'communication' | 'sewage' + infrastructure_kind: str + name: str | None + source_tag: str | None + distance_m: float + geometry_geojson: dict[str, Any] + + +class UtilityInfrastructureSummary(BaseModel): + total_features: int + nearest_distance_m: float | None + # Карта вид сети → расстояние до ближайшего объекта данного вида (м), либо null. + nearest_by_kind: dict[str, float | None] + + +class UtilityInfrastructureResponse(BaseModel): + features: list[UtilityInfrastructureFeature] + summary: UtilityInfrastructureSummary + + # ── NSPD Risk Zones schemas (issue #94 TIER 3) ─────────────────────────────── diff --git a/backend/app/services/site_finder/utility_infrastructure_loader.py b/backend/app/services/site_finder/utility_infrastructure_loader.py new file mode 100644 index 00000000..c0859ed9 --- /dev/null +++ b/backend/app/services/site_finder/utility_infrastructure_loader.py @@ -0,0 +1,390 @@ +"""Загрузчик OSM инженерных сетей («инженерные сети») из Overpass API (#1746). + +No-B2B open-data путь: тянет ЛЭП / подстанции / трубопроводы (газ/вода) / ЦТП / +вышки связи через Overpass для bbox ЕКБ и UPSERT-ит в +``osm_utility_infrastructure_ekb``. Живой тест дал 3081+ utility-элементов ЕКБ. + +Структура зеркалит ``noise_loader.py``: httpx-клиент с явным UA + таймаутом, +rate-limit ~1 req/s (Overpass usage policy), геометрия way → LineString / +node → Point, per-element SAVEPOINT при UPSERT (битый элемент не валит батч). + +Запускается раз в неделю через Celery beat. Геометрия включает И линии (ЛЭП, +трубопроводы — way), И точки (опоры, подстанции, узлы ВКХ, ЦТП, вышки — node). +""" + +import asyncio +import json +import logging + +import httpx +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.core.db import SessionLocal +from app.services.site_finder.noise_loader import ( + EKB_BBOX, + _way_to_linestring_wkt, +) + +logger = logging.getLogger(__name__) + +OVERPASS_URL = "https://overpass-api.de/api/interpreter" + +# Overpass блокирует дефолтный User-Agent (406) — явный UA с контактом (как noise_loader). +_HEADERS = { + "User-Agent": "GenDesign-SiteFinder/1.0 (+https://gendsgn.ru)", + "Accept": "application/json", +} + +# Описание запросов: (overpass_key, overpass_value, el_type, infrastructure_kind). +# el_type: 'way' (out geom) | 'node' (out body) | 'nwr' (out geom, node+way). +# substance: для pipeline-веток газ/вода дополнительно фильтруем по substance, +# чтобы не смешивать водопровод с газопроводом — см. _SUBSTANCE_FILTER. +# Виды сети (infrastructure_kind): +# power | water | gas | heat | communication | sewage +_UTILITY_QUERIES: list[tuple[str, str, str, str]] = [ + # power — магистральные ЛЭП (линии) + опоры (точки) + подстанции (точка/площадка) + ("power", "line", "way", "power"), + ("power", "minor_line", "way", "power"), + ("power", "tower", "node", "power"), + ("power", "substation", "nwr", "power"), + # water — водозаборы/узлы ВКХ (точка/площадка) + магистраль pipeline substance=water + ("man_made", "water_works", "nwr", "water"), + ("man_made", "water_tower", "nwr", "water"), + ("pipeline", "water", "way", "water"), + # gas — магистральные газопроводы (way), pipeline substance=gas + ("pipeline", "gas", "way", "gas"), + ("man_made", "pipeline", "way", "gas"), # substance=gas фильтруется ниже + # heat — ЦТП / теплопункты (sparse — это нормально) + ("man_made", "heat_substation", "nwr", "heat"), + # communication — вышки связи (точка/площадка) tower:type=communication + ("man_made", "tower", "nwr", "communication"), + # sewage — магистральная канализация (pipeline substance=sewerage) + очистные + ("pipeline", "sewerage", "way", "sewage"), + ("man_made", "wastewater_plant", "nwr", "sewage"), +] + +# Для generic man_made=pipeline нужно различать вещество по тегу `substance` +# (или legacy `type`): иначе под газ попадут все трубы. Ключ — (key, value), +# значение — ожидаемый substance. Элементы без совпадения substance пропускаем. +_SUBSTANCE_FILTER: dict[tuple[str, str], frozenset[str]] = { + ("man_made", "pipeline"): frozenset({"gas", "natural_gas", "cng"}), +} + +# tower:type=communication — для man_made=tower оставляем только вышки связи +# (иначе под communication попадут смотровые/водонапорные башни). +_TOWER_TYPE_REQUIRED: dict[tuple[str, str], frozenset[str]] = { + ("man_made", "tower"): frozenset({"communication", "communications"}), +} + + +def _build_overpass_query(key: str, value: str, el_type: str) -> str: + """Запрос Overpass для одного типа инж.-сетевого элемента. + + way / nwr → ``out geom`` (нужны координаты узлов для LineString / way-площадок); + node → ``out body``. Зеркалит noise_loader._build_overpass_query. + """ + south, west, north, east = EKB_BBOX + bbox = f"({south},{west},{north},{east})" + if el_type == "nwr": + return f"[out:json][timeout:30];" f'nwr["{key}"="{value}"]{bbox};' f"out geom;" + if el_type == "way": + return f"[out:json][timeout:30];" f'way["{key}"="{value}"]{bbox};' f"out geom;" + return f"[out:json][timeout:30];" f'node["{key}"="{value}"]{bbox};' f"out body;" + + +def _passes_tag_filters(key: str, value: str, tags: dict) -> bool: + """Доп-фильтр по тегам элемента (substance / tower:type). + + Для generic ``man_made=pipeline`` оставляем только газовые трубы (substance), + для ``man_made=tower`` — только вышки связи (tower:type). Остальные ключи + проходят без фильтра. + """ + substance_ok = _SUBSTANCE_FILTER.get((key, value)) + if substance_ok is not None: + substance = (tags.get("substance") or tags.get("type") or "").lower() + if substance not in substance_ok: + return False + tower_ok = _TOWER_TYPE_REQUIRED.get((key, value)) + if tower_ok is not None: + tower_type = (tags.get("tower:type") or "").lower() + if tower_type not in tower_ok: + return False + return True + + +async def fetch_overpass_utility() -> list[dict]: + """Запрашивает Overpass API для всех инж.-сетевых типов ЕКБ. + + Возвращает enriched list dict'ов с доп-полями: + ``_infrastructure_kind``, ``_source_tag`` — для UPSERT. + Между запросами sleep 1с (Overpass usage policy), как в noise_loader. + """ + all_elements: list[dict] = [] + async with httpx.AsyncClient(timeout=60, headers=_HEADERS) as client: + for key, value, el_type, kind in _UTILITY_QUERIES: + query = _build_overpass_query(key, value, el_type) + try: + r = await client.post(OVERPASS_URL, data={"data": query}) + r.raise_for_status() + elements: list[dict] = r.json().get("elements", []) + logger.info( + "Overpass utility: %s=%s (%s) → %d elements", + key, + value, + kind, + len(elements), + ) + for el in elements: + el["_infrastructure_kind"] = kind + el["_source_tag"] = f"{key}={value}" + el["_query_key"] = key + el["_query_value"] = value + all_elements.extend(elements) + except Exception as e: + logger.warning("Overpass utility failed for %s=%s: %s", key, value, e) + await asyncio.sleep(1.0) + logger.info("Overpass utility: total %d elements", len(all_elements)) + return all_elements + + +def sync_utility_infrastructure_to_db() -> dict[str, int]: + """UPSERT инж.сетей из Overpass в osm_utility_infrastructure_ekb. + + UPSERT по UNIQUE(osm_type, osm_id). Геометрия: + - way с nodes → LineString WKT через ST_GeomFromText(..., 4326) + - node → ST_SetSRID(ST_MakePoint(lon, lat), 4326) + Per-element SAVEPOINT — битый элемент не валит весь weekly-sync. + + Returns: счётчики fetched/inserted/updated/skipped + per-kind counts. + """ + elements = asyncio.run(fetch_overpass_utility()) + return _upsert_elements(elements) + + +def _upsert_elements(elements: list[dict]) -> dict[str, int]: + """UPSERT уже полученных Overpass-элементов в таблицу (выделено для тестов). + + Принимает enriched-элементы (с ``_infrastructure_kind`` / ``_source_tag``), + строит геометрию и UPSERT-ит под per-element SAVEPOINT. Возвращает счётчики. + """ + inserted = 0 + updated = 0 + skipped = 0 + fetched = len(elements) + by_kind: dict[str, int] = {} + + db = SessionLocal() + try: + for el in elements: + key: str = el.get("_query_key", "") + value: str = el.get("_query_value", "") + tags: dict = el.get("tags") or {} + + # substance / tower:type фильтры (generic man_made=pipeline / tower). + if not _passes_tag_filters(key, value, tags): + skipped += 1 + continue + + osm_id: int = el["id"] + osm_type: str = el["type"] # 'way' | 'node' + kind: str = el.get("_infrastructure_kind", "power") + source_tag: str | None = el.get("_source_tag") + name: str | None = tags.get("name") + + geom_sql: str + geom_params: dict + if osm_type == "way": + nodes: list[dict] = el.get("geometry") or [] + wkt = _way_to_linestring_wkt(nodes) + if wkt is None: + # way-площадка (закрытый контур) тоже даёт LineString из geometry — + # если узлов <2, пропускаем (битый/частичный bbox-обрезок). + skipped += 1 + continue + geom_sql = "ST_GeomFromText(:wkt, 4326)" + geom_params = {"wkt": wkt} + elif osm_type == "node": + lat = el.get("lat") + lon = el.get("lon") + if lat is None or lon is None: + skipped += 1 + continue + geom_sql = "ST_SetSRID(ST_MakePoint(:lon, :lat), 4326)" + geom_params = {"lat": lat, "lon": lon} + else: + # relation — skip (сложная геометрия, не нужна сейчас) + skipped += 1 + continue + + params: dict = { + "osm_id": osm_id, + "osm_type": osm_type, + "infrastructure_kind": kind, + "name": name, + "source_tag": source_tag, + "tags": json.dumps(tags, ensure_ascii=False), + **geom_params, + } + + try: + with db.begin_nested(): # SAVEPOINT — откат только этой записи + result = db.execute( + text(f""" + INSERT INTO osm_utility_infrastructure_ekb + (osm_type, osm_id, infrastructure_kind, name, geom, + source_tag, tags, fetched_at) + VALUES ( + :osm_type, :osm_id, :infrastructure_kind, :name, + {geom_sql}, + :source_tag, CAST(:tags AS jsonb), NOW() + ) + ON CONFLICT (osm_type, osm_id) DO UPDATE + SET infrastructure_kind = EXCLUDED.infrastructure_kind, + name = EXCLUDED.name, + geom = EXCLUDED.geom, + source_tag = EXCLUDED.source_tag, + tags = EXCLUDED.tags, + fetched_at = NOW() + RETURNING (xmax = 0) AS is_insert + """), + params, + ).scalar() + if result: + inserted += 1 + else: + updated += 1 + by_kind[kind] = by_kind.get(kind, 0) + 1 + except Exception as e: + # Дефектный элемент (битая геометрия и т.п.) не должен валить + # весь weekly-sync. SAVEPOINT откатывает только эту строку. + logger.warning( + "utility_sync upsert failed for %s/%s (kind=%s): %s", + osm_type, + osm_id, + kind, + e, + ) + skipped += 1 + + db.commit() + except Exception as e: + db.rollback() + logger.exception("utility_sync: unexpected error, outer tx rolled back: %s", e) + raise + finally: + db.close() + + result_dict: dict[str, int] = { + "fetched": fetched, + "inserted": inserted, + "updated": updated, + "skipped": skipped, + } + # per-kind counts (power/water/gas/heat/communication/sewage) — для отчёта populate. + for k, c in sorted(by_kind.items()): + result_dict[f"kind_{k}"] = c + logger.info("utility_sync done: %s", result_dict) + return result_dict + + +async def load_utility_infrastructure(db: Session | None = None) -> dict[str, int]: + """Async-обёртка для прямого вызова из async-контекста (#1746 spec). + + Тянет Overpass и UPSERT-ит в таблицу. ``db`` принимается для сигнатурной + совместимости со спекой, но UPSERT идёт через собственную SessionLocal + (как noise/poi sync) — Overpass-fetch async, запись в БД sync. Возвращает + те же per-kind counts, что ``sync_utility_infrastructure_to_db``. + """ + elements = await fetch_overpass_utility() + return _upsert_elements(elements) + + +def get_nearby_utility_infrastructure( + db: Session, + cad_num: str, + radius_m: int = 500, + limit: int = 200, +) -> dict: + """Инженерные сети из OSM в radius_m от центроида участка (#1746 endpoint). + + Источник: ``osm_utility_infrastructure_ekb`` (наполняется weekly-sync из + Overpass). Расстояние считается от ЦЕНТРОИДА участка через ST_DWithin / + ST_Distance в geography. Возвращает feature-list + summary с ближайшими + расстояниями по видам сети. + + Args: + db: SQLAlchemy session. + cad_num: кадастровый номер участка (e.g. '66:41:0204016:10'). + radius_m: радиус поиска, м (caller валидирует диапазон). + limit: максимум фич в ответе (ближайшие). + + Returns: + {"features": [...], "summary": {...}} + + Raises: + ValueError: участок не найден в БД (нет geom). + """ + # Импорт здесь, чтобы избежать циклического импорта на module-load + # (quarter_dump_lookup → noise_loader → ... ). + from app.services.site_finder.quarter_dump_lookup import _get_parcel_wkt + + parcel_wkt = _get_parcel_wkt(db, cad_num) + if parcel_wkt is None: + raise ValueError(f"Участок {cad_num!r} не найден в БД") + + rows = ( + db.execute( + text(""" + SELECT osm_id, osm_type, infrastructure_kind, name, source_tag, + ST_Distance( + u.geom::geography, + ST_Centroid(ST_GeomFromText(:wkt, 4326))::geography + ) AS distance_m, + ST_AsGeoJSON(u.geom) AS geojson + FROM osm_utility_infrastructure_ekb u + WHERE ST_DWithin( + u.geom::geography, + ST_Centroid(ST_GeomFromText(:wkt, 4326))::geography, + CAST(:radius_m AS float) + ) + ORDER BY distance_m ASC + LIMIT :lim + """), + {"wkt": parcel_wkt, "radius_m": radius_m, "lim": limit}, + ) + .mappings() + .all() + ) + + features: list[dict] = [] + nearest_by_kind: dict[str, float | None] = {} + nearest_overall: float | None = None + for r in rows: + dist = round(float(r["distance_m"]), 1) + kind = r["infrastructure_kind"] + # rows отсортированы по distance ASC → первое попадание вида = ближайшее. + if kind not in nearest_by_kind: + nearest_by_kind[kind] = dist + if nearest_overall is None: + nearest_overall = dist + features.append( + { + "osm_id": int(r["osm_id"]), + "osm_type": r["osm_type"], + "infrastructure_kind": kind, + "name": r["name"], + "source_tag": r["source_tag"], + "distance_m": dist, + "geometry_geojson": json.loads(r["geojson"]) if r["geojson"] else {}, + } + ) + + return { + "features": features, + "summary": { + "total_features": len(features), + "nearest_distance_m": nearest_overall, + "nearest_by_kind": nearest_by_kind, + }, + } diff --git a/backend/app/workers/beat_schedule.py b/backend/app/workers/beat_schedule.py index cb9f8e6b..557842b6 100644 --- a/backend/app/workers/beat_schedule.py +++ b/backend/app/workers/beat_schedule.py @@ -274,6 +274,15 @@ def build_beat_schedule() -> dict: "options": {"queue": "celery"}, } + # OSM инженерные сети (#1746) sync — еженедельно в понедельник в 04:00 МСК + # (через 30 мин после noise-sync, чтобы не нагружать Overpass одновременно). + # No-B2B open-data источник (Overpass): ЛЭП/подстанции/трубопроводы/ЦТП/вышки. + schedule["utility-infrastructure-sync-weekly"] = { + "task": "tasks.utility_infrastructure_sync.sync_osm_utility_infrastructure_ekb", + "schedule": _parse_cron("0 4 * * mon"), + "options": {"queue": "celery"}, + } + # #105 Phase 4: ЕКБ РНС/РВЭ — ежемесячно 1-го числа в 05:00 МСК. # Celery conf.timezone=Europe/Moscow → crontab трактуется в МСК (#1233). schedule["ekburg-permits-monthly"] = { diff --git a/backend/app/workers/celery_app.py b/backend/app/workers/celery_app.py index 9418adc2..08d56989 100644 --- a/backend/app/workers/celery_app.py +++ b/backend/app/workers/celery_app.py @@ -57,6 +57,7 @@ celery_app = Celery( "app.workers.tasks.nspd_sync", "app.workers.tasks.poi_sync", "app.workers.tasks.noise_sync", + "app.workers.tasks.utility_infrastructure_sync", "app.workers.tasks.pzz_sync", "app.workers.tasks.scrape_cadastre", "app.workers.tasks.ekburg_permits_sync", diff --git a/backend/app/workers/tasks/utility_infrastructure_sync.py b/backend/app/workers/tasks/utility_infrastructure_sync.py new file mode 100644 index 00000000..d9a09a03 --- /dev/null +++ b/backend/app/workers/tasks/utility_infrastructure_sync.py @@ -0,0 +1,59 @@ +"""Celery task: еженедельная синхронизация OSM инженерных сетей для site-finder (#1746).""" + +import logging + +from sqlalchemy import text + +from app.workers.celery_app import celery_app + +logger = logging.getLogger(__name__) + + +def _log_breadcrumb(level: str, message: str) -> None: + """Записать строку в nspd_geo_log (job_id=NULL, stage='utility_sync'). + + Persistent breadcrumb для диагностики без SSH (как poi_sync / noise_sync). + """ + try: + from app.core.db import SessionLocal + + db = SessionLocal() + try: + db.execute( + text( + "INSERT INTO nspd_geo_log (job_id, level, stage, message) " + "VALUES (NULL, :lvl, 'utility_sync', :msg)" + ), + {"lvl": level, "msg": message[:500]}, + ) + db.commit() + finally: + db.close() + except Exception as e: + logger.warning("utility_sync breadcrumb failed: %s", e) + + +@celery_app.task( + name="tasks.utility_infrastructure_sync.sync_osm_utility_infrastructure_ekb", + queue="celery", +) +def sync_osm_utility_infrastructure_ekb() -> dict: + """Еженедельная синхронизация OSM инж.сетей из Overpass в osm_utility_infrastructure_ekb. + + Запускается через beat (понедельник 04:00 МСК — через 30 мин после noise-sync). + No-B2B open-data источник (Overpass). Persistent breadcrumbs в nspd_geo_log. + """ + _log_breadcrumb("info", "task started") + try: + from app.services.site_finder.utility_infrastructure_loader import ( + sync_utility_infrastructure_to_db, + ) + + result = sync_utility_infrastructure_to_db() + logger.info("utility_sync done: %s", result) + _log_breadcrumb("info", f"done: {result}") + return result + except Exception as e: + logger.exception("utility_sync failed: %s", e) + _log_breadcrumb("error", f"{type(e).__name__}: {e}") + raise diff --git a/backend/tests/api/v1/test_parcel_utility_infrastructure.py b/backend/tests/api/v1/test_parcel_utility_infrastructure.py new file mode 100644 index 00000000..d54b2a18 --- /dev/null +++ b/backend/tests/api/v1/test_parcel_utility_infrastructure.py @@ -0,0 +1,136 @@ +"""Тесты для GET /{cad_num}/utility-infrastructure (#1746). + +FastAPI TestClient + mock сервиса get_nearby_utility_infrastructure — без Postgres. +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import patch + +import pytest +from fastapi.testclient import TestClient + +from app.main import app + +_VALID_CAD = "66:41:0204016:10" + + +def _make_feature( + osm_id: int = 1, + kind: str = "power", + distance_m: float = 120.5, + osm_type: str = "way", +) -> dict[str, Any]: + return { + "osm_id": osm_id, + "osm_type": osm_type, + "infrastructure_kind": kind, + "name": "ЛЭП-110", + "source_tag": "power=line", + "distance_m": distance_m, + "geometry_geojson": {"type": "LineString", "coordinates": [[60.6, 56.8], [60.61, 56.81]]}, + } + + +def _make_response( + features: list[dict[str, Any]] | None = None, + nearest: float | None = 120.5, + nearest_by_kind: dict[str, float | None] | None = None, +) -> dict[str, Any]: + if features is None: + features = [_make_feature()] + if nearest_by_kind is None: + nearest_by_kind = {"power": 120.5} + return { + "features": features, + "summary": { + "total_features": len(features), + "nearest_distance_m": nearest, + "nearest_by_kind": nearest_by_kind, + }, + } + + +def test_utility_infra_returns_features() -> None: + """Нормальный ответ: фичи + summary с ближайшими по видам сети.""" + full = _make_response( + features=[ + _make_feature(1, "power", 80.0, "way"), + _make_feature(2, "water", 150.0, "node"), + ], + nearest=80.0, + nearest_by_kind={"power": 80.0, "water": 150.0}, + ) + with patch( + "app.api.v1.parcels.get_nearby_utility_infrastructure", + return_value=full, + ): + client = TestClient(app) + r = client.get(f"/api/v1/parcels/{_VALID_CAD}/utility-infrastructure") + + assert r.status_code == 200, r.text + body = r.json() + assert body["summary"]["total_features"] == 2 + assert body["summary"]["nearest_distance_m"] == pytest.approx(80.0) + assert body["summary"]["nearest_by_kind"]["power"] == pytest.approx(80.0) + assert body["summary"]["nearest_by_kind"]["water"] == pytest.approx(150.0) + assert len(body["features"]) == 2 + assert body["features"][0]["infrastructure_kind"] == "power" + assert body["features"][0]["geometry_geojson"]["type"] == "LineString" + + +def test_utility_infra_empty() -> None: + """Нет сетей в радиусе → пустой список, total=0, nearest=null.""" + empty = _make_response(features=[], nearest=None, nearest_by_kind={}) + with patch( + "app.api.v1.parcels.get_nearby_utility_infrastructure", + return_value=empty, + ): + client = TestClient(app) + r = client.get(f"/api/v1/parcels/{_VALID_CAD}/utility-infrastructure") + + assert r.status_code == 200, r.text + body = r.json() + assert body["features"] == [] + assert body["summary"]["total_features"] == 0 + assert body["summary"]["nearest_distance_m"] is None + assert body["summary"]["nearest_by_kind"] == {} + + +def test_utility_infra_parcel_not_found_404() -> None: + """Участок не найден (ValueError из сервиса) → 404.""" + with patch( + "app.api.v1.parcels.get_nearby_utility_infrastructure", + side_effect=ValueError("Участок '66:41:9999999:1' не найден в БД"), + ): + client = TestClient(app) + r = client.get("/api/v1/parcels/66:41:9999999:1/utility-infrastructure") + + assert r.status_code == 404 + assert "не найден" in r.json()["detail"] + + +def test_utility_infra_passes_radius() -> None: + """radius_m из query передаётся в сервис.""" + with patch("app.api.v1.parcels.get_nearby_utility_infrastructure") as mock_svc: + mock_svc.return_value = _make_response() + client = TestClient(app) + r = client.get(f"/api/v1/parcels/{_VALID_CAD}/utility-infrastructure?radius_m=1000") + assert r.status_code == 200 + call = mock_svc.call_args + assert call[0][2] == 1000 or call[1].get("radius_m") == 1000 + + +def test_utility_infra_radius_out_of_range_422() -> None: + """radius_m=10 (< min 50) → 422.""" + client = TestClient(app) + r = client.get(f"/api/v1/parcels/{_VALID_CAD}/utility-infrastructure?radius_m=10") + assert r.status_code == 422 + + +def test_utility_infra_radius_too_large_422() -> None: + """radius_m=5000 (> max 2000) → 422.""" + client = TestClient(app) + r = client.get(f"/api/v1/parcels/{_VALID_CAD}/utility-infrastructure?radius_m=5000") + assert r.status_code == 422 diff --git a/backend/tests/test_utility_infrastructure_loader.py b/backend/tests/test_utility_infrastructure_loader.py new file mode 100644 index 00000000..b068ceae --- /dev/null +++ b/backend/tests/test_utility_infrastructure_loader.py @@ -0,0 +1,284 @@ +"""Unit tests для utility_infrastructure_loader — OSM инженерные сети (#1746). + +Mock-based / pure — без реального Overpass и БД. Покрывает: +- _build_overpass_query: node / way / nwr варианты + bbox ЕКБ +- _passes_tag_filters: substance-фильтр (man_made=pipeline) + tower:type (man_made=tower) +- _upsert_elements: парсинг way→LineString / node→Point, классификация kind, + per-element SAVEPOINT (битый элемент не валит батч), per-kind counts +""" + +from __future__ import annotations + +import json +from contextlib import contextmanager +from typing import Any + +from app.services.site_finder import utility_infrastructure_loader as ul + +# ── _build_overpass_query ───────────────────────────────────────────────────── + + +def test_build_query_node() -> None: + q = ul._build_overpass_query("power", "tower", "node") + assert q.startswith("[out:json][timeout:30];") + assert 'node["power"="tower"]' in q + assert "out body;" in q + + +def test_build_query_way_uses_out_geom() -> None: + """way → out geom (нужны координаты узлов для LineString ЛЭП/трубы).""" + q = ul._build_overpass_query("power", "line", "way") + assert 'way["power"="line"]' in q + assert "out geom;" in q + + +def test_build_query_nwr_for_points() -> None: + """nwr ловит и node, и way (точка/площадка подстанции), с out geom.""" + q = ul._build_overpass_query("power", "substation", "nwr") + assert 'nwr["power"="substation"]' in q + assert "out geom;" in q + + +def test_build_query_bbox_present() -> None: + """Bbox ЕКБ присутствует во всех вариантах запроса.""" + q = ul._build_overpass_query("pipeline", "gas", "way") + assert "(56.7,60.5,56.95,60.75)" in q + + +# ── _UTILITY_QUERIES coverage ───────────────────────────────────────────────── + + +def test_all_required_kinds_present() -> None: + """Все 6 видов сети покрыты запросами.""" + kinds = {kind for _k, _v, _t, kind in ul._UTILITY_QUERIES} + for expected in ("power", "water", "gas", "heat", "communication", "sewage"): + assert expected in kinds, f"вид {expected!r} должен быть в _UTILITY_QUERIES" + + +def test_power_line_is_way_and_tower_is_node() -> None: + """ЛЭП = way (линия), опора = node (точка).""" + by_kv = {(k, v): t for k, v, t, _ in ul._UTILITY_QUERIES} + assert by_kv[("power", "line")] == "way" + assert by_kv[("power", "tower")] == "node" + assert by_kv[("power", "substation")] == "nwr" + + +# ── _passes_tag_filters ─────────────────────────────────────────────────────── + + +def test_pipeline_substance_gas_passes() -> None: + assert ul._passes_tag_filters("man_made", "pipeline", {"substance": "gas"}) + + +def test_pipeline_substance_non_gas_filtered() -> None: + """man_made=pipeline с substance=oil НЕ проходит под gas.""" + assert not ul._passes_tag_filters("man_made", "pipeline", {"substance": "oil"}) + + +def test_pipeline_legacy_type_tag_fallback() -> None: + """legacy `type=gas` (без substance) тоже распознаётся.""" + assert ul._passes_tag_filters("man_made", "pipeline", {"type": "gas"}) + + +def test_tower_communication_passes() -> None: + assert ul._passes_tag_filters("man_made", "tower", {"tower:type": "communication"}) + + +def test_tower_non_communication_filtered() -> None: + """Водонапорная башня (tower:type=water) НЕ проходит под communication.""" + assert not ul._passes_tag_filters("man_made", "tower", {"tower:type": "water"}) + + +def test_unfiltered_key_always_passes() -> None: + """Ключи без substance/tower фильтра проходят как есть.""" + assert ul._passes_tag_filters("power", "line", {}) + + +# ── _upsert_elements (fake DB) ──────────────────────────────────────────────── + + +class _Scalar: + def __init__(self, value: Any) -> None: + self._value = value + + def scalar(self) -> Any: + return self._value + + +class _FakeDB: + """Мини-фейк SQLAlchemy session: пишет executed, отдаёт is_insert через scalar().""" + + def __init__(self, *, fail_ids: set[int] | None = None) -> None: + self.executed: list[tuple[Any, dict[str, Any] | None]] = [] + self.committed = False + self.closed = False + self.rolled_back = False + self._fail_ids = fail_ids or set() + + @contextmanager + def begin_nested(self) -> Any: + yield + + def execute(self, sql: Any, params: dict[str, Any] | None = None) -> Any: + self.executed.append((sql, params)) + if params is not None and params.get("osm_id") in self._fail_ids: + raise RuntimeError("bad geometry") + return _Scalar(True) # is_insert=True + + def commit(self) -> None: + self.committed = True + + def rollback(self) -> None: + self.rolled_back = True + + def close(self) -> None: + self.closed = True + + +def _way(osm_id: int, kind: str, key: str, value: str, nodes: list[dict]) -> dict: + return { + "id": osm_id, + "type": "way", + "geometry": nodes, + "tags": {"name": f"way-{osm_id}"}, + "_infrastructure_kind": kind, + "_source_tag": f"{key}={value}", + "_query_key": key, + "_query_value": value, + } + + +def _node(osm_id: int, kind: str, key: str, value: str, tags: dict | None = None) -> dict: + return { + "id": osm_id, + "type": "node", + "lat": 56.83, + "lon": 60.6, + "tags": tags or {"name": f"node-{osm_id}"}, + "_infrastructure_kind": kind, + "_source_tag": f"{key}={value}", + "_query_key": key, + "_query_value": value, + } + + +_TWO_NODES = [{"lat": 56.8, "lon": 60.6}, {"lat": 56.81, "lon": 60.61}] + + +def test_upsert_mixed_power_water_gas(monkeypatch: Any) -> None: + """Power line (way), substation (node), water_works (node), gas pipeline (way).""" + fake_db = _FakeDB() + monkeypatch.setattr(ul, "SessionLocal", lambda: fake_db) + + elements = [ + _way(1, "power", "power", "line", _TWO_NODES), + _node(2, "power", "power", "substation"), + _node(3, "water", "man_made", "water_works"), + _way(4, "gas", "pipeline", "gas", _TWO_NODES), + ] + result = ul._upsert_elements(elements) + + assert result["fetched"] == 4 + assert result["inserted"] == 4 + assert result["skipped"] == 0 + assert result["kind_power"] == 2 + assert result["kind_water"] == 1 + assert result["kind_gas"] == 1 + assert fake_db.committed + assert fake_db.closed + assert len(fake_db.executed) == 4 + + +def test_upsert_way_geometry_is_linestring(monkeypatch: Any) -> None: + """way → WKT LINESTRING передаётся в params (geom-build из noise_loader).""" + fake_db = _FakeDB() + monkeypatch.setattr(ul, "SessionLocal", lambda: fake_db) + + ul._upsert_elements([_way(1, "power", "power", "line", _TWO_NODES)]) + _, params = fake_db.executed[0] + assert params is not None + assert params["wkt"] == "LINESTRING(60.6 56.8, 60.61 56.81)" + assert params["infrastructure_kind"] == "power" + assert params["source_tag"] == "power=line" + + +def test_upsert_node_geometry_is_point(monkeypatch: Any) -> None: + """node → lat/lon передаются для ST_MakePoint.""" + fake_db = _FakeDB() + monkeypatch.setattr(ul, "SessionLocal", lambda: fake_db) + + ul._upsert_elements([_node(2, "power", "power", "substation")]) + _, params = fake_db.executed[0] + assert params is not None + assert params["lat"] == 56.83 + assert params["lon"] == 60.6 + assert "wkt" not in params + + +def test_upsert_skips_way_with_too_few_nodes(monkeypatch: Any) -> None: + """way с <2 узлами (битый bbox-обрезок) пропускается, не валит батч.""" + fake_db = _FakeDB() + monkeypatch.setattr(ul, "SessionLocal", lambda: fake_db) + + elements = [ + _way(1, "power", "power", "line", [{"lat": 56.8, "lon": 60.6}]), # 1 узел + _node(2, "power", "power", "substation"), + ] + result = ul._upsert_elements(elements) + assert result["skipped"] == 1 + assert result["inserted"] == 1 + assert len(fake_db.executed) == 1 # только node дошёл до UPSERT + + +def test_upsert_substance_filter_skips_non_gas_pipeline(monkeypatch: Any) -> None: + """man_made=pipeline substance=oil пропускается (не газ).""" + fake_db = _FakeDB() + monkeypatch.setattr(ul, "SessionLocal", lambda: fake_db) + + oil = _way(1, "gas", "man_made", "pipeline", _TWO_NODES) + oil["tags"] = {"substance": "oil"} + gas = _way(2, "gas", "man_made", "pipeline", _TWO_NODES) + gas["tags"] = {"substance": "gas"} + + result = ul._upsert_elements([oil, gas]) + assert result["skipped"] == 1 + assert result["inserted"] == 1 + assert result["kind_gas"] == 1 + + +def test_upsert_savepoint_isolates_bad_element(monkeypatch: Any) -> None: + """Битый элемент (execute бросает) → SAVEPOINT откатывает только его, батч жив.""" + fake_db = _FakeDB(fail_ids={2}) + monkeypatch.setattr(ul, "SessionLocal", lambda: fake_db) + + elements = [ + _node(1, "power", "power", "substation"), + _node(2, "power", "power", "substation"), # упадёт + _node(3, "water", "man_made", "water_works"), + ] + result = ul._upsert_elements(elements) + assert result["inserted"] == 2 + assert result["skipped"] == 1 + assert fake_db.committed # outer tx НЕ откатан + assert not fake_db.rolled_back + + +def test_upsert_tags_serialized_json(monkeypatch: Any) -> None: + """tags сериализуются в JSON-строку (для CAST(:tags AS jsonb)).""" + fake_db = _FakeDB() + monkeypatch.setattr(ul, "SessionLocal", lambda: fake_db) + + ul._upsert_elements([_node(2, "power", "power", "substation", {"name": "ТП-5"})]) + _, params = fake_db.executed[0] + assert params is not None + assert json.loads(params["tags"]) == {"name": "ТП-5"} + assert params["name"] == "ТП-5" + + +def test_upsert_empty_returns_zero(monkeypatch: Any) -> None: + fake_db = _FakeDB() + monkeypatch.setattr(ul, "SessionLocal", lambda: fake_db) + result = ul._upsert_elements([]) + assert result["fetched"] == 0 + assert result["inserted"] == 0 + assert len(fake_db.executed) == 0 diff --git a/data/sql/170_osm_utility_infrastructure_ekb.sql b/data/sql/170_osm_utility_infrastructure_ekb.sql new file mode 100644 index 00000000..9082eef2 --- /dev/null +++ b/data/sql/170_osm_utility_infrastructure_ekb.sql @@ -0,0 +1,53 @@ +-- 170_osm_utility_infrastructure_ekb.sql +-- Context: Таблица инженерных сетей («инженерные сети») Екатеринбурга из OSM (#1746). +-- No-B2B open-data путь: Overpass API. Хранит магистральные линии +-- (ЛЭП, трубопроводы — way → LineString) И точки/узлы (опоры ЛЭП, +-- подстанции, водозаборы, ЦТП, вышки связи — node → Point) для быстрых +-- ST_DWithin-запросов «инж.сети в радиусе участка». +-- Дополняет osm_noise_sources_ekb (там utility-срез завязан на noise-loader); +-- этот слой — самостоятельный, типизированный по infrastructure_kind. +-- Source: Overpass API (power/water/gas/heat/communication queries). +-- Geometry: mixed (LineString для way / Point для node) -> geometry(Geometry, 4326) +-- Dependencies: PostGIS (уже установлен в prod) +-- Deploy order: standalone, нет зависимостей от других таблиц +-- Idempotent: да (IF NOT EXISTS / IF NOT EXISTS на индексах) + +BEGIN; + +CREATE TABLE IF NOT EXISTS osm_utility_infrastructure_ekb ( + id BIGSERIAL PRIMARY KEY, + osm_id BIGINT NOT NULL, + osm_type TEXT NOT NULL, -- 'node' | 'way' | 'relation' + -- Нормализованный вид инженерной сети: + -- 'power' | 'water' | 'gas' | 'heat' | 'communication' | 'sewage' + infrastructure_kind TEXT NOT NULL, + name TEXT, + geom GEOMETRY(Geometry, 4326) NOT NULL, -- линии И точки + source_tag TEXT, -- OSM key=value, по которому элемент пойман + tags JSONB DEFAULT '{}'::jsonb, + fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (osm_type, osm_id) +); + +-- GIST индекс для ST_DWithin / ST_Intersects запросов (инж.сети в радиусе участка) +CREATE INDEX IF NOT EXISTS osm_utility_infrastructure_ekb_geom_idx + ON osm_utility_infrastructure_ekb USING GIST (geom); + +-- Индекс по виду сети для фильтрации (power/water/gas/...) +CREATE INDEX IF NOT EXISTS osm_utility_infrastructure_ekb_kind_idx + ON osm_utility_infrastructure_ekb (infrastructure_kind); + +COMMENT ON TABLE osm_utility_infrastructure_ekb IS + 'Инженерные сети Екатеринбурга из OSM (power/water/gas/heat/communication/sewage). ' + 'No-B2B open-data источник: Overpass API. Линии (ЛЭП/трубопроводы) и узлы ' + '(подстанции/водозаборы/ЦТП/вышки связи) для site-finder utility-infrastructure. ' + 'Маппинг OSM-тегов -> infrastructure_kind: ' + 'backend/app/services/site_finder/utility_infrastructure_loader.py'; + +COMMENT ON COLUMN osm_utility_infrastructure_ekb.infrastructure_kind IS + 'Нормализованный вид сети: power | water | gas | heat | communication | sewage.'; + +COMMENT ON COLUMN osm_utility_infrastructure_ekb.source_tag IS + 'OSM key=value, по которому элемент пойман (например power=line, pipeline=gas).'; + +COMMIT; diff --git a/frontend/src/lib/api-types.ts b/frontend/src/lib/api-types.ts index 38a4f459..a8ad8484 100644 --- a/frontend/src/lib/api-types.ts +++ b/frontend/src/lib/api-types.ts @@ -339,6 +339,35 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/parcels/{cad_num}/utility-infrastructure": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Инженерные сети из OSM вблизи участка (#1746) + * @description Инженерные сети (ЛЭП/подстанции/трубопроводы/ЦТП/вышки) из OSM вблизи участка. + * + * No-B2B open-data источник: osm_utility_infrastructure_ekb (наполняется weekly-sync + * из Overpass). Возвращает фичи в радиусе от центроида участка (ST_DWithin) + + * summary с ближайшими расстояниями по видам сети. + * + * Query params: + * radius_m: радиус поиска, 50–2000 м (default 500). + * + * Если участок не найден в БД (нет geom) → 404. + */ + get: operations["get_parcel_utility_infrastructure_api_v1_parcels__cad_num__utility_infrastructure_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/parcels/{cad_num}/isochrones": { parameters: { query?: never; @@ -4714,6 +4743,42 @@ export interface components { /** Deny Paths */ deny_paths: string[]; }; + /** UtilityInfrastructureFeature */ + UtilityInfrastructureFeature: { + /** Osm Id */ + osm_id: number; + /** Osm Type */ + osm_type: string; + /** Infrastructure Kind */ + infrastructure_kind: string; + /** Name */ + name: string | null; + /** Source Tag */ + source_tag: string | null; + /** Distance M */ + distance_m: number; + /** Geometry Geojson */ + geometry_geojson: { + [key: string]: unknown; + }; + }; + /** UtilityInfrastructureResponse */ + UtilityInfrastructureResponse: { + /** Features */ + features: components["schemas"]["UtilityInfrastructureFeature"][]; + summary: components["schemas"]["UtilityInfrastructureSummary"]; + }; + /** UtilityInfrastructureSummary */ + UtilityInfrastructureSummary: { + /** Total Features */ + total_features: number; + /** Nearest Distance M */ + nearest_distance_m: number | null; + /** Nearest By Kind */ + nearest_by_kind: { + [key: string]: number | null; + }; + }; /** ValidationError */ ValidationError: { /** Location */ @@ -5206,6 +5271,39 @@ export interface operations { }; }; }; + get_parcel_utility_infrastructure_api_v1_parcels__cad_num__utility_infrastructure_get: { + parameters: { + query?: { + radius_m?: number; + }; + header?: never; + path: { + cad_num: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UtilityInfrastructureResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_isochrones_api_v1_parcels__cad_num__isochrones_get: { parameters: { query?: {