fdced94626
- Add _GRID_WALK_LAYERS frozenset (territorial_zones, red_lines, engineering_structures, zouit_*, risk_*) - _fetch_layer dispatch: area layers → get_features_in_bbox_grid(grid_n=7, step_m=50) [foundation из PR #247], EGRN layers (parcels, buildings) → legacy get_features_in_bbox - Enrich engineering_structures features с classified_kind через classify_engineering_kind (PR #247 classifier) - Per-layer logger.info heartbeat (method + count) для прогресса - Bump harvest_quarter soft_time_limit 120 → 600s (grid-walk: 11 layers × 49 cells × ~70ms ≈ 40s + retries margin) - TestFetchLayerDispatch (3 tests): 34 passed, 1 skipped Pre-push deep-code-reviewer: APPROVE (1 medium nit — opportunity_oopt не в grid-walk, защищено через include_opportunity=False default; follow-up issue рекомендован).
449 lines
20 KiB
Python
449 lines
20 KiB
Python
"""Unit-тесты для NSPDClient.get_features_in_bbox_grid + classify_engineering_kind.
|
||
|
||
Sub-PR A foundation (#126): grid-walk — НЕ live HTTP (all mocked).
|
||
|
||
Запуск:
|
||
cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -v
|
||
|
||
Live integration (помечены @pytest.mark.integration — пропускаются в CI):
|
||
cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -m integration -s
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from typing import Any, ClassVar
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.nspd_client import NSPDClient, NSPDFeature
|
||
from app.services.scrapers.nspd_denorm import classify_engineering_kind
|
||
|
||
# ── Helpers ────────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _make_bulk_feature(feature_id: str | int | None, props: dict[str, Any]) -> MagicMock:
|
||
"""Создать mock NSPDBulkFeature с нужными полями."""
|
||
feat = MagicMock()
|
||
feat.id = feature_id
|
||
feat.geometry = None
|
||
feat.properties = props
|
||
return feat
|
||
|
||
|
||
# ── Grid-walk count tests ──────────────────────────────────────────────────────
|
||
|
||
|
||
class TestGetFeaturesInBboxGrid:
|
||
"""Тесты grid-walk метода (без live NSPD)."""
|
||
|
||
# bbox 700×700м в EPSG:3857 (типичный квартал ЕКБ)
|
||
BBOX: tuple[float, float, float, float] = (
|
||
6_600_000.0,
|
||
7_700_000.0,
|
||
6_600_700.0,
|
||
7_700_700.0,
|
||
)
|
||
|
||
def _patch_bulk_client(self, return_features: list[list[Any]]) -> tuple[Any, AsyncMock]:
|
||
"""Патч NSPDBulkClient.wms_feature_info.
|
||
|
||
return_features: список ответов — по одному на каждый вызов wms_feature_info
|
||
(в порядке вызовов). Если список короче числа calls — последний элемент
|
||
повторяется.
|
||
"""
|
||
call_idx: list[int] = [0]
|
||
|
||
async def _side_effect(*args: Any, **kwargs: Any) -> list[Any]:
|
||
idx = min(call_idx[0], len(return_features) - 1)
|
||
call_idx[0] += 1
|
||
return return_features[idx]
|
||
|
||
mock_wms = AsyncMock(side_effect=_side_effect)
|
||
return mock_wms, mock_wms
|
||
|
||
def test_grid_n7_calls_49_times(self) -> None:
|
||
"""grid_n=7 → ровно 49 вызовов wms_feature_info."""
|
||
call_count: list[int] = [0]
|
||
|
||
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
|
||
call_count[0] += 1
|
||
return []
|
||
|
||
mock_client_instance = AsyncMock()
|
||
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
|
||
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
|
||
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
|
||
|
||
with patch(
|
||
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
|
||
return_value=mock_client_instance,
|
||
):
|
||
client = NSPDClient()
|
||
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=7, step_m=1.0)
|
||
|
||
assert call_count[0] == 49, f"Ожидали 49 вызовов, получили {call_count[0]}"
|
||
assert result == []
|
||
|
||
def test_grid_n3_calls_9_times(self) -> None:
|
||
"""grid_n=3 → ровно 9 вызовов."""
|
||
call_count: list[int] = [0]
|
||
|
||
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
|
||
call_count[0] += 1
|
||
return []
|
||
|
||
mock_client_instance = AsyncMock()
|
||
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
|
||
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
|
||
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
|
||
|
||
with patch(
|
||
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
|
||
return_value=mock_client_instance,
|
||
):
|
||
client = NSPDClient()
|
||
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=3, step_m=1.0)
|
||
|
||
assert call_count[0] == 9
|
||
assert result == []
|
||
|
||
def test_dedup_same_feature_id(self) -> None:
|
||
"""Одинаковые feature_id из соседних ячеек → одна запись."""
|
||
feat_a = _make_bulk_feature("feat-001", {"cad_num": "66:41:001:1"})
|
||
feat_b = _make_bulk_feature("feat-001", {"cad_num": "66:41:001:1"}) # дубликат
|
||
feat_c = _make_bulk_feature("feat-002", {"cad_num": "66:41:001:2"})
|
||
|
||
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
|
||
return [feat_a, feat_b, feat_c]
|
||
|
||
mock_client_instance = AsyncMock()
|
||
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
|
||
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
|
||
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
|
||
|
||
with patch(
|
||
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
|
||
return_value=mock_client_instance,
|
||
):
|
||
client = NSPDClient()
|
||
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0)
|
||
|
||
# 4 cells × 3 features = 12 raw, но feature_id уникальных 2
|
||
feature_ids = [f.feature_id for f in result]
|
||
assert "feat-001" in feature_ids
|
||
assert "feat-002" in feature_ids
|
||
# нет дубликатов
|
||
assert len(feature_ids) == len(set(feature_ids))
|
||
|
||
def test_dedup_by_cad_num_when_no_feature_id(self) -> None:
|
||
"""Если feature_id=None — дедупликация по cad_num."""
|
||
feat_a = _make_bulk_feature(None, {"cad_num": "66:41:0000001:100"})
|
||
feat_b = _make_bulk_feature(None, {"cad_num": "66:41:0000001:100"}) # дубликат
|
||
|
||
call_n: list[int] = [0]
|
||
|
||
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
|
||
call_n[0] += 1
|
||
# Первая ячейка возвращает feat_a, вторая — feat_b
|
||
return [feat_a] if call_n[0] == 1 else [feat_b]
|
||
|
||
mock_client_instance = AsyncMock()
|
||
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
|
||
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
|
||
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
|
||
|
||
with patch(
|
||
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
|
||
return_value=mock_client_instance,
|
||
):
|
||
client = NSPDClient()
|
||
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0)
|
||
|
||
cad_nums = [f.properties.get("cad_num") for f in result]
|
||
assert cad_nums.count("66:41:0000001:100") == 1, "Дубликат не дедуплицирован"
|
||
|
||
def test_error_in_one_cell_does_not_abort(self) -> None:
|
||
"""Ошибка в одной ячейке не останавливает весь grid-walk."""
|
||
good_feat = _make_bulk_feature("feat-ok", {"cad_num": "66:41:001:1"})
|
||
call_n: list[int] = [0]
|
||
|
||
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
|
||
call_n[0] += 1
|
||
if call_n[0] == 1:
|
||
raise RuntimeError("Simulated cell error")
|
||
return [good_feat]
|
||
|
||
mock_client_instance = AsyncMock()
|
||
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
|
||
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
|
||
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
|
||
|
||
with patch(
|
||
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
|
||
return_value=mock_client_instance,
|
||
):
|
||
client = NSPDClient()
|
||
# Не должно бросать исключение
|
||
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0)
|
||
|
||
# 4 cells: 1 error + 3 good_feat → 1 unique feature
|
||
assert any(f.feature_id == "feat-ok" for f in result)
|
||
|
||
def test_returns_nspd_feature_instances(self) -> None:
|
||
"""Метод возвращает list[NSPDFeature] а не NSPDBulkFeature."""
|
||
bulk_feat = _make_bulk_feature("feat-xyz", {"cad_num": "66:41:001:1"})
|
||
|
||
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
|
||
return [bulk_feat]
|
||
|
||
mock_client_instance = AsyncMock()
|
||
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
|
||
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
|
||
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
|
||
|
||
with patch(
|
||
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
|
||
return_value=mock_client_instance,
|
||
):
|
||
client = NSPDClient()
|
||
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=1, step_m=1.0)
|
||
|
||
assert all(isinstance(f, NSPDFeature) for f in result)
|
||
|
||
def test_auto_reduce_grid_for_small_bbox(self) -> None:
|
||
"""Маленький bbox + большой grid_n + большой step_m → grid_n уменьшается."""
|
||
# bbox 60×60м, step_m=50, grid_n=7 → effective_n=min(7, int(60/50), int(60/50)) = 1
|
||
small_bbox: tuple[float, float, float, float] = (
|
||
6_600_000.0,
|
||
7_700_000.0,
|
||
6_600_060.0,
|
||
7_700_060.0,
|
||
)
|
||
call_count: list[int] = [0]
|
||
|
||
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
|
||
call_count[0] += 1
|
||
return []
|
||
|
||
mock_client_instance = AsyncMock()
|
||
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
|
||
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
|
||
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
|
||
|
||
with patch(
|
||
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
|
||
return_value=mock_client_instance,
|
||
):
|
||
client = NSPDClient()
|
||
client.get_features_in_bbox_grid(36328, small_bbox, grid_n=7, step_m=50.0)
|
||
|
||
# effective_n = 1 → 1 ячейка
|
||
assert call_count[0] == 1
|
||
|
||
|
||
# ── Classifier tests ───────────────────────────────────────────────────────────
|
||
|
||
|
||
class TestClassifyEngineeringKind:
|
||
"""Smoke-тесты классификатора инженерных сооружений."""
|
||
|
||
@pytest.mark.parametrize(
|
||
"props, expected",
|
||
[
|
||
# gas
|
||
({"params_name": "Газопровод высокого давления"}, "gas"),
|
||
({"name": "Газопровод ПЭ SDR11 160мм"}, "gas"),
|
||
({"params_purpose": "Газоснабжение"}, "gas"),
|
||
({"name": "ГРП №12"}, "gas"),
|
||
# electric
|
||
({"name": "КЛ 10 кВ ТП 64102"}, "electric"),
|
||
({"params_name": "ВЛ-10кВ Ф-14"}, "electric"),
|
||
({"name": "ТП 1234"}, "electric"),
|
||
({"params_purpose": "Электроэнергетика и связь"}, "electric"),
|
||
({"name": "Подстанция 110/10 кВ"}, "electric"),
|
||
# water
|
||
({"params_purpose": "Водопровод хозбытовой"}, "water"),
|
||
({"name": "Водовод Ду300"}, "water"),
|
||
({"params_name": "Сеть водоснабжения"}, "water"),
|
||
# heat
|
||
({"params_name": "Тепловая сеть"}, "heat"),
|
||
({"name": "Теплосеть квартал 24"}, "heat"),
|
||
({"params_purpose": "Теплоснабжение жилых домов"}, "heat"),
|
||
({"name": "ТЭЦ-4 отпайка"}, "heat"),
|
||
# sewage
|
||
({"name": "Канализация"}, "sewage"),
|
||
({"params_name": "Сеть канализации Ду200"}, "sewage"),
|
||
({"name": "Ливневая канализация"}, "sewage"),
|
||
# other
|
||
({"name": "Объект не классифицирован"}, "other"),
|
||
({}, "other"),
|
||
({"params_purpose": None}, "other"),
|
||
],
|
||
)
|
||
def test_classify(self, props: dict[str, Any], expected: str) -> None:
|
||
assert (
|
||
classify_engineering_kind(props) == expected
|
||
), f"props={props!r} → expected {expected!r}"
|
||
|
||
def test_field_priority_params_name_over_purpose(self) -> None:
|
||
"""params_name проверяется раньше purpose."""
|
||
props = {
|
||
"params_name": "Газопровод", # → gas
|
||
"params_purpose": "Водоснабжение", # → water (если бы проверялось первым)
|
||
}
|
||
# gas-паттерн найдётся в combined строке первым по порядку _ENGINEERING_PATTERNS
|
||
assert classify_engineering_kind(props) == "gas"
|
||
|
||
def test_case_insensitive(self) -> None:
|
||
"""Паттерны case-insensitive."""
|
||
assert classify_engineering_kind({"name": "ГАЗОПРОВОД ВЫСОКОГО ДАВЛЕНИЯ"}) == "gas"
|
||
assert classify_engineering_kind({"name": "канализация бытовая"}) == "sewage"
|
||
|
||
|
||
# ── Sub-PR B: _fetch_layer dispatch tests ─────────────────────────────────────
|
||
|
||
|
||
class TestFetchLayerDispatch:
|
||
"""Проверяем что _fetch_layer внутри search_by_quarter правильно диспатчит
|
||
area-слои на get_features_in_bbox_grid, а EGRN-слои — на legacy get_features_in_bbox.
|
||
|
||
Моки: search_by_cad и get_features_in_bbox_grid / get_features_in_bbox
|
||
через patch, без live HTTP.
|
||
"""
|
||
|
||
# Минимальный bbox квартала (Web Mercator), который возвращает search_by_cad
|
||
_MOCK_GEOM: ClassVar[dict[str, Any]] = {
|
||
"type": "Polygon",
|
||
"coordinates": [
|
||
[
|
||
[6_600_000.0, 7_700_000.0],
|
||
[6_600_700.0, 7_700_000.0],
|
||
[6_600_700.0, 7_700_700.0],
|
||
[6_600_000.0, 7_700_700.0],
|
||
[6_600_000.0, 7_700_000.0],
|
||
]
|
||
],
|
||
}
|
||
|
||
def _make_search_result(self) -> MagicMock:
|
||
"""Mock NSPDSearchResult с первым feature, у которого есть geometry."""
|
||
feat = MagicMock(spec=NSPDFeature)
|
||
feat.geometry = self._MOCK_GEOM
|
||
feat.properties = {}
|
||
feat.feature_id = "q-001"
|
||
result = MagicMock()
|
||
result.first = feat
|
||
return result
|
||
|
||
def test_fetch_layer_uses_grid_walk_for_area_layers(self) -> None:
|
||
"""territorial_zones (area layer) → вызывает get_features_in_bbox_grid."""
|
||
client = NSPDClient()
|
||
mock_search_result = self._make_search_result()
|
||
|
||
with (
|
||
patch.object(client, "search_by_cad", return_value=mock_search_result),
|
||
patch.object(client, "get_features_in_bbox_grid", return_value=[]) as mock_grid,
|
||
patch.object(client, "get_features_in_bbox", return_value=[]) as mock_legacy,
|
||
):
|
||
client.search_by_quarter(
|
||
"66:41:0303161",
|
||
include_zouit=False,
|
||
include_risks=False,
|
||
include_opportunity=False,
|
||
)
|
||
|
||
# territorial_zones, red_lines, engineering_structures — все три должны
|
||
# быть запрошены через grid-walk
|
||
called_layer_ids = [call.args[0] for call in mock_grid.call_args_list]
|
||
from app.services.scrapers.nspd_client import LAYERS
|
||
|
||
assert (
|
||
LAYERS["territorial_zones"] in called_layer_ids
|
||
), "territorial_zones должен использовать grid-walk"
|
||
assert LAYERS["red_lines"] in called_layer_ids, "red_lines должен использовать grid-walk"
|
||
assert (
|
||
LAYERS["engineering_structures"] in called_layer_ids
|
||
), "engineering_structures должен использовать grid-walk"
|
||
# parcels и buildings — legacy, не grid
|
||
called_legacy_ids = [call.args[0] for call in mock_legacy.call_args_list]
|
||
assert LAYERS["parcels"] in called_legacy_ids, "parcels должен идти через legacy"
|
||
assert LAYERS["buildings"] in called_legacy_ids, "buildings должен идти через legacy"
|
||
|
||
def test_fetch_layer_uses_legacy_for_egrn_layers(self) -> None:
|
||
"""parcels и buildings → вызывают legacy get_features_in_bbox (не grid)."""
|
||
client = NSPDClient()
|
||
mock_search_result = self._make_search_result()
|
||
|
||
with (
|
||
patch.object(client, "search_by_cad", return_value=mock_search_result),
|
||
patch.object(client, "get_features_in_bbox_grid", return_value=[]) as mock_grid,
|
||
patch.object(client, "get_features_in_bbox", return_value=[]) as mock_legacy,
|
||
):
|
||
client.search_by_quarter(
|
||
"66:41:0303161",
|
||
include_zouit=False,
|
||
include_risks=False,
|
||
include_opportunity=False,
|
||
)
|
||
|
||
from app.services.scrapers.nspd_client import LAYERS
|
||
|
||
called_legacy_ids = [call.args[0] for call in mock_legacy.call_args_list]
|
||
called_grid_ids = [call.args[0] for call in mock_grid.call_args_list]
|
||
|
||
assert LAYERS["parcels"] in called_legacy_ids
|
||
assert LAYERS["buildings"] in called_legacy_ids
|
||
# EGRN layers должны НЕ попасть в grid-walk
|
||
assert LAYERS["parcels"] not in called_grid_ids, "parcels не должен использовать grid"
|
||
assert LAYERS["buildings"] not in called_grid_ids, "buildings не должен использовать grid"
|
||
|
||
def test_engineering_structures_classified_kind_enriched(self) -> None:
|
||
"""engineering_structures features получают classified_kind в properties."""
|
||
from app.services.scrapers.nspd_client import LAYERS
|
||
|
||
# Создаём feature с mutable properties (как реальный NSPDFeature)
|
||
eng_feat = MagicMock(spec=NSPDFeature)
|
||
eng_feat.feature_id = "eng-001"
|
||
eng_feat.geometry = None
|
||
eng_feat.properties = {"params_name": "Газопровод высокого давления"}
|
||
|
||
client = NSPDClient()
|
||
mock_search_result = self._make_search_result()
|
||
|
||
def _grid_side_effect(layer_id: int, bbox: Any, **kwargs: Any) -> list[Any]:
|
||
if layer_id == LAYERS["engineering_structures"]:
|
||
return [eng_feat]
|
||
return []
|
||
|
||
with (
|
||
patch.object(client, "search_by_cad", return_value=mock_search_result),
|
||
patch.object(client, "get_features_in_bbox_grid", side_effect=_grid_side_effect),
|
||
patch.object(client, "get_features_in_bbox", return_value=[]),
|
||
):
|
||
dump = client.search_by_quarter(
|
||
"66:41:0303161",
|
||
include_zouit=False,
|
||
include_risks=False,
|
||
include_opportunity=False,
|
||
)
|
||
|
||
assert len(dump.engineering_structures) == 1
|
||
enriched = dump.engineering_structures[0]
|
||
got = enriched.properties.get("classified_kind")
|
||
assert got == "gas", f"Ожидали classified_kind='gas', получили {got!r}"
|
||
|
||
|
||
# ── Integration marker (skip in CI) ──────────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.integration
|
||
def test_live_nspd_grid_walk_skipped() -> None:
|
||
"""Placeholder для ручного запуска live NSPD grid-walk.
|
||
|
||
Запуск:
|
||
cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -m integration -s
|
||
|
||
При запуске — делает реальные HTTP-запросы к nspd.gov.ru.
|
||
"""
|
||
pytest.skip("Live NSPD integration test — запускать вручную с -m integration")
|