Light1YT
42ed1f8a93
No-B2B open-data путь (Overpass) для инженерных сетей ЕКБ: ЛЭП/подстанции/
опоры, трубопроводы (газ/вода/канализация), ЦТП, вышки связи.
- Миграция 100_osm_utility_infrastructure_ekb.sql: таблица + GIST-индекс,
UNIQUE(osm_type, osm_id) для идемпотентного UPSERT, geom(Geometry,4326)
(линии И точки).
- Loader utility_infrastructure_loader.py: Overpass-запросы (rate-limit 1 req/s),
классификация в infrastructure_kind (power/water/gas/heat/communication/sewage),
per-element SAVEPOINT UPSERT. Зеркалит noise_loader (геом-хелперы переиспользованы).
- Celery task + weekly beat (понедельник 04:00 МСК, после noise-sync).
- Endpoint GET /api/v1/parcels/{cad}/utility-infrastructure: инж.сети в радиусе
центроида участка (ST_DWithin) + summary по видам сети. Зеркалит connection-points.
- Тесты: loader (Overpass-парсинг/UPSERT/SAVEPOINT/фильтры) + endpoint (mock DB).
- Codegen: api-types.ts регенерирован (новый endpoint + 3 схемы).
284 lines
11 KiB
Python
284 lines
11 KiB
Python
"""Unit tests для utility_infrastructure_loader — OSM инженерные сети (#1746).
|
||
|
||
Mock-based / pure — без реального Overpass и БД. Покрывает:
|
||
- _build_overpass_query: node / way / nwr варианты + bbox ЕКБ
|
||
- _passes_tag_filters: substance-фильтр (man_made=pipeline) + tower:type (man_made=tower)
|
||
- _upsert_elements: парсинг way→LineString / node→Point, классификация kind,
|
||
per-element SAVEPOINT (битый элемент не валит батч), per-kind counts
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
from contextlib import contextmanager
|
||
from typing import Any
|
||
|
||
from app.services.site_finder import utility_infrastructure_loader as ul
|
||
|
||
# ── _build_overpass_query ─────────────────────────────────────────────────────
|
||
|
||
|
||
def test_build_query_node() -> None:
|
||
q = ul._build_overpass_query("power", "tower", "node")
|
||
assert q.startswith("[out:json][timeout:30];")
|
||
assert 'node["power"="tower"]' in q
|
||
assert "out body;" in q
|
||
|
||
|
||
def test_build_query_way_uses_out_geom() -> None:
|
||
"""way → out geom (нужны координаты узлов для LineString ЛЭП/трубы)."""
|
||
q = ul._build_overpass_query("power", "line", "way")
|
||
assert 'way["power"="line"]' in q
|
||
assert "out geom;" in q
|
||
|
||
|
||
def test_build_query_nwr_for_points() -> None:
|
||
"""nwr ловит и node, и way (точка/площадка подстанции), с out geom."""
|
||
q = ul._build_overpass_query("power", "substation", "nwr")
|
||
assert 'nwr["power"="substation"]' in q
|
||
assert "out geom;" in q
|
||
|
||
|
||
def test_build_query_bbox_present() -> None:
|
||
"""Bbox ЕКБ присутствует во всех вариантах запроса."""
|
||
q = ul._build_overpass_query("pipeline", "gas", "way")
|
||
assert "(56.7,60.5,56.95,60.75)" in q
|
||
|
||
|
||
# ── _UTILITY_QUERIES coverage ─────────────────────────────────────────────────
|
||
|
||
|
||
def test_all_required_kinds_present() -> None:
|
||
"""Все 6 видов сети покрыты запросами."""
|
||
kinds = {kind for _k, _v, _t, kind in ul._UTILITY_QUERIES}
|
||
for expected in ("power", "water", "gas", "heat", "communication", "sewage"):
|
||
assert expected in kinds, f"вид {expected!r} должен быть в _UTILITY_QUERIES"
|
||
|
||
|
||
def test_power_line_is_way_and_tower_is_node() -> None:
|
||
"""ЛЭП = way (линия), опора = node (точка)."""
|
||
by_kv = {(k, v): t for k, v, t, _ in ul._UTILITY_QUERIES}
|
||
assert by_kv[("power", "line")] == "way"
|
||
assert by_kv[("power", "tower")] == "node"
|
||
assert by_kv[("power", "substation")] == "nwr"
|
||
|
||
|
||
# ── _passes_tag_filters ───────────────────────────────────────────────────────
|
||
|
||
|
||
def test_pipeline_substance_gas_passes() -> None:
|
||
assert ul._passes_tag_filters("man_made", "pipeline", {"substance": "gas"})
|
||
|
||
|
||
def test_pipeline_substance_non_gas_filtered() -> None:
|
||
"""man_made=pipeline с substance=oil НЕ проходит под gas."""
|
||
assert not ul._passes_tag_filters("man_made", "pipeline", {"substance": "oil"})
|
||
|
||
|
||
def test_pipeline_legacy_type_tag_fallback() -> None:
|
||
"""legacy `type=gas` (без substance) тоже распознаётся."""
|
||
assert ul._passes_tag_filters("man_made", "pipeline", {"type": "gas"})
|
||
|
||
|
||
def test_tower_communication_passes() -> None:
|
||
assert ul._passes_tag_filters("man_made", "tower", {"tower:type": "communication"})
|
||
|
||
|
||
def test_tower_non_communication_filtered() -> None:
|
||
"""Водонапорная башня (tower:type=water) НЕ проходит под communication."""
|
||
assert not ul._passes_tag_filters("man_made", "tower", {"tower:type": "water"})
|
||
|
||
|
||
def test_unfiltered_key_always_passes() -> None:
|
||
"""Ключи без substance/tower фильтра проходят как есть."""
|
||
assert ul._passes_tag_filters("power", "line", {})
|
||
|
||
|
||
# ── _upsert_elements (fake DB) ────────────────────────────────────────────────
|
||
|
||
|
||
class _Scalar:
|
||
def __init__(self, value: Any) -> None:
|
||
self._value = value
|
||
|
||
def scalar(self) -> Any:
|
||
return self._value
|
||
|
||
|
||
class _FakeDB:
|
||
"""Мини-фейк SQLAlchemy session: пишет executed, отдаёт is_insert через scalar()."""
|
||
|
||
def __init__(self, *, fail_ids: set[int] | None = None) -> None:
|
||
self.executed: list[tuple[Any, dict[str, Any] | None]] = []
|
||
self.committed = False
|
||
self.closed = False
|
||
self.rolled_back = False
|
||
self._fail_ids = fail_ids or set()
|
||
|
||
@contextmanager
|
||
def begin_nested(self) -> Any:
|
||
yield
|
||
|
||
def execute(self, sql: Any, params: dict[str, Any] | None = None) -> Any:
|
||
self.executed.append((sql, params))
|
||
if params is not None and params.get("osm_id") in self._fail_ids:
|
||
raise RuntimeError("bad geometry")
|
||
return _Scalar(True) # is_insert=True
|
||
|
||
def commit(self) -> None:
|
||
self.committed = True
|
||
|
||
def rollback(self) -> None:
|
||
self.rolled_back = True
|
||
|
||
def close(self) -> None:
|
||
self.closed = True
|
||
|
||
|
||
def _way(osm_id: int, kind: str, key: str, value: str, nodes: list[dict]) -> dict:
|
||
return {
|
||
"id": osm_id,
|
||
"type": "way",
|
||
"geometry": nodes,
|
||
"tags": {"name": f"way-{osm_id}"},
|
||
"_infrastructure_kind": kind,
|
||
"_source_tag": f"{key}={value}",
|
||
"_query_key": key,
|
||
"_query_value": value,
|
||
}
|
||
|
||
|
||
def _node(osm_id: int, kind: str, key: str, value: str, tags: dict | None = None) -> dict:
|
||
return {
|
||
"id": osm_id,
|
||
"type": "node",
|
||
"lat": 56.83,
|
||
"lon": 60.6,
|
||
"tags": tags or {"name": f"node-{osm_id}"},
|
||
"_infrastructure_kind": kind,
|
||
"_source_tag": f"{key}={value}",
|
||
"_query_key": key,
|
||
"_query_value": value,
|
||
}
|
||
|
||
|
||
_TWO_NODES = [{"lat": 56.8, "lon": 60.6}, {"lat": 56.81, "lon": 60.61}]
|
||
|
||
|
||
def test_upsert_mixed_power_water_gas(monkeypatch: Any) -> None:
|
||
"""Power line (way), substation (node), water_works (node), gas pipeline (way)."""
|
||
fake_db = _FakeDB()
|
||
monkeypatch.setattr(ul, "SessionLocal", lambda: fake_db)
|
||
|
||
elements = [
|
||
_way(1, "power", "power", "line", _TWO_NODES),
|
||
_node(2, "power", "power", "substation"),
|
||
_node(3, "water", "man_made", "water_works"),
|
||
_way(4, "gas", "pipeline", "gas", _TWO_NODES),
|
||
]
|
||
result = ul._upsert_elements(elements)
|
||
|
||
assert result["fetched"] == 4
|
||
assert result["inserted"] == 4
|
||
assert result["skipped"] == 0
|
||
assert result["kind_power"] == 2
|
||
assert result["kind_water"] == 1
|
||
assert result["kind_gas"] == 1
|
||
assert fake_db.committed
|
||
assert fake_db.closed
|
||
assert len(fake_db.executed) == 4
|
||
|
||
|
||
def test_upsert_way_geometry_is_linestring(monkeypatch: Any) -> None:
|
||
"""way → WKT LINESTRING передаётся в params (geom-build из noise_loader)."""
|
||
fake_db = _FakeDB()
|
||
monkeypatch.setattr(ul, "SessionLocal", lambda: fake_db)
|
||
|
||
ul._upsert_elements([_way(1, "power", "power", "line", _TWO_NODES)])
|
||
_, params = fake_db.executed[0]
|
||
assert params is not None
|
||
assert params["wkt"] == "LINESTRING(60.6 56.8, 60.61 56.81)"
|
||
assert params["infrastructure_kind"] == "power"
|
||
assert params["source_tag"] == "power=line"
|
||
|
||
|
||
def test_upsert_node_geometry_is_point(monkeypatch: Any) -> None:
|
||
"""node → lat/lon передаются для ST_MakePoint."""
|
||
fake_db = _FakeDB()
|
||
monkeypatch.setattr(ul, "SessionLocal", lambda: fake_db)
|
||
|
||
ul._upsert_elements([_node(2, "power", "power", "substation")])
|
||
_, params = fake_db.executed[0]
|
||
assert params is not None
|
||
assert params["lat"] == 56.83
|
||
assert params["lon"] == 60.6
|
||
assert "wkt" not in params
|
||
|
||
|
||
def test_upsert_skips_way_with_too_few_nodes(monkeypatch: Any) -> None:
|
||
"""way с <2 узлами (битый bbox-обрезок) пропускается, не валит батч."""
|
||
fake_db = _FakeDB()
|
||
monkeypatch.setattr(ul, "SessionLocal", lambda: fake_db)
|
||
|
||
elements = [
|
||
_way(1, "power", "power", "line", [{"lat": 56.8, "lon": 60.6}]), # 1 узел
|
||
_node(2, "power", "power", "substation"),
|
||
]
|
||
result = ul._upsert_elements(elements)
|
||
assert result["skipped"] == 1
|
||
assert result["inserted"] == 1
|
||
assert len(fake_db.executed) == 1 # только node дошёл до UPSERT
|
||
|
||
|
||
def test_upsert_substance_filter_skips_non_gas_pipeline(monkeypatch: Any) -> None:
|
||
"""man_made=pipeline substance=oil пропускается (не газ)."""
|
||
fake_db = _FakeDB()
|
||
monkeypatch.setattr(ul, "SessionLocal", lambda: fake_db)
|
||
|
||
oil = _way(1, "gas", "man_made", "pipeline", _TWO_NODES)
|
||
oil["tags"] = {"substance": "oil"}
|
||
gas = _way(2, "gas", "man_made", "pipeline", _TWO_NODES)
|
||
gas["tags"] = {"substance": "gas"}
|
||
|
||
result = ul._upsert_elements([oil, gas])
|
||
assert result["skipped"] == 1
|
||
assert result["inserted"] == 1
|
||
assert result["kind_gas"] == 1
|
||
|
||
|
||
def test_upsert_savepoint_isolates_bad_element(monkeypatch: Any) -> None:
|
||
"""Битый элемент (execute бросает) → SAVEPOINT откатывает только его, батч жив."""
|
||
fake_db = _FakeDB(fail_ids={2})
|
||
monkeypatch.setattr(ul, "SessionLocal", lambda: fake_db)
|
||
|
||
elements = [
|
||
_node(1, "power", "power", "substation"),
|
||
_node(2, "power", "power", "substation"), # упадёт
|
||
_node(3, "water", "man_made", "water_works"),
|
||
]
|
||
result = ul._upsert_elements(elements)
|
||
assert result["inserted"] == 2
|
||
assert result["skipped"] == 1
|
||
assert fake_db.committed # outer tx НЕ откатан
|
||
assert not fake_db.rolled_back
|
||
|
||
|
||
def test_upsert_tags_serialized_json(monkeypatch: Any) -> None:
|
||
"""tags сериализуются в JSON-строку (для CAST(:tags AS jsonb))."""
|
||
fake_db = _FakeDB()
|
||
monkeypatch.setattr(ul, "SessionLocal", lambda: fake_db)
|
||
|
||
ul._upsert_elements([_node(2, "power", "power", "substation", {"name": "ТП-5"})])
|
||
_, params = fake_db.executed[0]
|
||
assert params is not None
|
||
assert json.loads(params["tags"]) == {"name": "ТП-5"}
|
||
assert params["name"] == "ТП-5"
|
||
|
||
|
||
def test_upsert_empty_returns_zero(monkeypatch: Any) -> None:
|
||
fake_db = _FakeDB()
|
||
monkeypatch.setattr(ul, "SessionLocal", lambda: fake_db)
|
||
result = ul._upsert_elements([])
|
||
assert result["fetched"] == 0
|
||
assert result["inserted"] == 0
|
||
assert len(fake_db.executed) == 0
|