"""Unit-тесты для NSPDClient.get_features_in_bbox_grid + classify_engineering_kind. Sub-PR A foundation (#126): grid-walk — НЕ live HTTP (all mocked). Запуск: cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -v Live integration (помечены @pytest.mark.integration — пропускаются в CI): cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -m integration -s """ from __future__ import annotations from typing import Any, ClassVar from unittest.mock import AsyncMock, MagicMock, patch import pytest from app.services.scrapers.nspd_client import NSPDClient, NSPDFeature from app.services.scrapers.nspd_denorm import classify_engineering_kind # ── Helpers ──────────────────────────────────────────────────────────────────── def _make_bulk_feature(feature_id: str | int | None, props: dict[str, Any]) -> MagicMock: """Создать mock NSPDBulkFeature с нужными полями.""" feat = MagicMock() feat.id = feature_id feat.geometry = None feat.properties = props return feat # ── Grid-walk count tests ────────────────────────────────────────────────────── class TestGetFeaturesInBboxGrid: """Тесты grid-walk метода (без live NSPD).""" # bbox 700×700м в EPSG:3857 (типичный квартал ЕКБ) BBOX: tuple[float, float, float, float] = ( 6_600_000.0, 7_700_000.0, 6_600_700.0, 7_700_700.0, ) def _patch_bulk_client(self, return_features: list[list[Any]]) -> tuple[Any, AsyncMock]: """Патч NSPDBulkClient.wms_feature_info. return_features: список ответов — по одному на каждый вызов wms_feature_info (в порядке вызовов). Если список короче числа calls — последний элемент повторяется. """ call_idx: list[int] = [0] async def _side_effect(*args: Any, **kwargs: Any) -> list[Any]: idx = min(call_idx[0], len(return_features) - 1) call_idx[0] += 1 return return_features[idx] mock_wms = AsyncMock(side_effect=_side_effect) return mock_wms, mock_wms def test_grid_n7_calls_49_times(self) -> None: """grid_n=7 → ровно 49 вызовов wms_feature_info.""" call_count: list[int] = [0] async def _wms(*args: Any, **kwargs: Any) -> list[Any]: call_count[0] += 1 return [] mock_client_instance = AsyncMock() mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms) mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance) mock_client_instance.__aexit__ = AsyncMock(return_value=None) with patch( "app.scrapers.nspd_bulk_client.NSPDBulkClient", return_value=mock_client_instance, ): client = NSPDClient() result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=7, step_m=1.0) assert call_count[0] == 49, f"Ожидали 49 вызовов, получили {call_count[0]}" assert result == [] def test_grid_n3_calls_9_times(self) -> None: """grid_n=3 → ровно 9 вызовов.""" call_count: list[int] = [0] async def _wms(*args: Any, **kwargs: Any) -> list[Any]: call_count[0] += 1 return [] mock_client_instance = AsyncMock() mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms) mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance) mock_client_instance.__aexit__ = AsyncMock(return_value=None) with patch( "app.scrapers.nspd_bulk_client.NSPDBulkClient", return_value=mock_client_instance, ): client = NSPDClient() result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=3, step_m=1.0) assert call_count[0] == 9 assert result == [] def test_dedup_same_feature_id(self) -> None: """Одинаковые feature_id из соседних ячеек → одна запись.""" feat_a = _make_bulk_feature("feat-001", {"cad_num": "66:41:001:1"}) feat_b = _make_bulk_feature("feat-001", {"cad_num": "66:41:001:1"}) # дубликат feat_c = _make_bulk_feature("feat-002", {"cad_num": "66:41:001:2"}) async def _wms(*args: Any, **kwargs: Any) -> list[Any]: return [feat_a, feat_b, feat_c] mock_client_instance = AsyncMock() mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms) mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance) mock_client_instance.__aexit__ = AsyncMock(return_value=None) with patch( "app.scrapers.nspd_bulk_client.NSPDBulkClient", return_value=mock_client_instance, ): client = NSPDClient() result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0) # 4 cells × 3 features = 12 raw, но feature_id уникальных 2 feature_ids = [f.feature_id for f in result] assert "feat-001" in feature_ids assert "feat-002" in feature_ids # нет дубликатов assert len(feature_ids) == len(set(feature_ids)) def test_dedup_by_cad_num_when_no_feature_id(self) -> None: """Если feature_id=None — дедупликация по cad_num.""" feat_a = _make_bulk_feature(None, {"cad_num": "66:41:0000001:100"}) feat_b = _make_bulk_feature(None, {"cad_num": "66:41:0000001:100"}) # дубликат call_n: list[int] = [0] async def _wms(*args: Any, **kwargs: Any) -> list[Any]: call_n[0] += 1 # Первая ячейка возвращает feat_a, вторая — feat_b return [feat_a] if call_n[0] == 1 else [feat_b] mock_client_instance = AsyncMock() mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms) mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance) mock_client_instance.__aexit__ = AsyncMock(return_value=None) with patch( "app.scrapers.nspd_bulk_client.NSPDBulkClient", return_value=mock_client_instance, ): client = NSPDClient() result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0) cad_nums = [f.properties.get("cad_num") for f in result] assert cad_nums.count("66:41:0000001:100") == 1, "Дубликат не дедуплицирован" def test_error_in_one_cell_does_not_abort(self) -> None: """Ошибка в одной ячейке не останавливает весь grid-walk.""" good_feat = _make_bulk_feature("feat-ok", {"cad_num": "66:41:001:1"}) call_n: list[int] = [0] async def _wms(*args: Any, **kwargs: Any) -> list[Any]: call_n[0] += 1 if call_n[0] == 1: raise RuntimeError("Simulated cell error") return [good_feat] mock_client_instance = AsyncMock() mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms) mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance) mock_client_instance.__aexit__ = AsyncMock(return_value=None) with patch( "app.scrapers.nspd_bulk_client.NSPDBulkClient", return_value=mock_client_instance, ): client = NSPDClient() # Не должно бросать исключение result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0) # 4 cells: 1 error + 3 good_feat → 1 unique feature assert any(f.feature_id == "feat-ok" for f in result) def test_returns_nspd_feature_instances(self) -> None: """Метод возвращает list[NSPDFeature] а не NSPDBulkFeature.""" bulk_feat = _make_bulk_feature("feat-xyz", {"cad_num": "66:41:001:1"}) async def _wms(*args: Any, **kwargs: Any) -> list[Any]: return [bulk_feat] mock_client_instance = AsyncMock() mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms) mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance) mock_client_instance.__aexit__ = AsyncMock(return_value=None) with patch( "app.scrapers.nspd_bulk_client.NSPDBulkClient", return_value=mock_client_instance, ): client = NSPDClient() result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=1, step_m=1.0) assert all(isinstance(f, NSPDFeature) for f in result) def test_auto_reduce_grid_for_small_bbox(self) -> None: """Маленький bbox + большой grid_n + большой step_m → grid_n уменьшается.""" # bbox 60×60м, step_m=50, grid_n=7 → effective_n=min(7, int(60/50), int(60/50)) = 1 small_bbox: tuple[float, float, float, float] = ( 6_600_000.0, 7_700_000.0, 6_600_060.0, 7_700_060.0, ) call_count: list[int] = [0] async def _wms(*args: Any, **kwargs: Any) -> list[Any]: call_count[0] += 1 return [] mock_client_instance = AsyncMock() mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms) mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance) mock_client_instance.__aexit__ = AsyncMock(return_value=None) with patch( "app.scrapers.nspd_bulk_client.NSPDBulkClient", return_value=mock_client_instance, ): client = NSPDClient() client.get_features_in_bbox_grid(36328, small_bbox, grid_n=7, step_m=50.0) # effective_n = 1 → 1 ячейка assert call_count[0] == 1 # ── Classifier tests ─────────────────────────────────────────────────────────── class TestClassifyEngineeringKind: """Smoke-тесты классификатора инженерных сооружений.""" @pytest.mark.parametrize( "props, expected", [ # gas ({"params_name": "Газопровод высокого давления"}, "gas"), ({"name": "Газопровод ПЭ SDR11 160мм"}, "gas"), ({"params_purpose": "Газоснабжение"}, "gas"), ({"name": "ГРП №12"}, "gas"), # electric ({"name": "КЛ 10 кВ ТП 64102"}, "electric"), ({"params_name": "ВЛ-10кВ Ф-14"}, "electric"), ({"name": "ТП 1234"}, "electric"), ({"params_purpose": "Электроэнергетика и связь"}, "electric"), ({"name": "Подстанция 110/10 кВ"}, "electric"), # water ({"params_purpose": "Водопровод хозбытовой"}, "water"), ({"name": "Водовод Ду300"}, "water"), ({"params_name": "Сеть водоснабжения"}, "water"), # heat ({"params_name": "Тепловая сеть"}, "heat"), ({"name": "Теплосеть квартал 24"}, "heat"), ({"params_purpose": "Теплоснабжение жилых домов"}, "heat"), ({"name": "ТЭЦ-4 отпайка"}, "heat"), # sewage ({"name": "Канализация"}, "sewage"), ({"params_name": "Сеть канализации Ду200"}, "sewage"), ({"name": "Ливневая канализация"}, "sewage"), # other ({"name": "Объект не классифицирован"}, "other"), ({}, "other"), ({"params_purpose": None}, "other"), ], ) def test_classify(self, props: dict[str, Any], expected: str) -> None: assert ( classify_engineering_kind(props) == expected ), f"props={props!r} → expected {expected!r}" def test_field_priority_params_name_over_purpose(self) -> None: """params_name проверяется раньше purpose.""" props = { "params_name": "Газопровод", # → gas "params_purpose": "Водоснабжение", # → water (если бы проверялось первым) } # gas-паттерн найдётся в combined строке первым по порядку _ENGINEERING_PATTERNS assert classify_engineering_kind(props) == "gas" def test_case_insensitive(self) -> None: """Паттерны case-insensitive.""" assert classify_engineering_kind({"name": "ГАЗОПРОВОД ВЫСОКОГО ДАВЛЕНИЯ"}) == "gas" assert classify_engineering_kind({"name": "канализация бытовая"}) == "sewage" # ── Sub-PR B: _fetch_layer dispatch tests ───────────────────────────────────── class TestFetchLayerDispatch: """Проверяем что _fetch_layer внутри search_by_quarter правильно диспатчит area-слои на get_features_in_bbox_grid, а EGRN-слои — на legacy get_features_in_bbox. Моки: search_by_cad и get_features_in_bbox_grid / get_features_in_bbox через patch, без live HTTP. """ # Минимальный bbox квартала (Web Mercator), который возвращает search_by_cad _MOCK_GEOM: ClassVar[dict[str, Any]] = { "type": "Polygon", "coordinates": [ [ [6_600_000.0, 7_700_000.0], [6_600_700.0, 7_700_000.0], [6_600_700.0, 7_700_700.0], [6_600_000.0, 7_700_700.0], [6_600_000.0, 7_700_000.0], ] ], } def _make_search_result(self) -> MagicMock: """Mock NSPDSearchResult с первым feature, у которого есть geometry.""" feat = MagicMock(spec=NSPDFeature) feat.geometry = self._MOCK_GEOM feat.properties = {} feat.feature_id = "q-001" result = MagicMock() result.first = feat return result def test_fetch_layer_uses_grid_walk_for_area_layers(self) -> None: """territorial_zones (area layer) → вызывает get_features_in_bbox_grid.""" client = NSPDClient() mock_search_result = self._make_search_result() with ( patch.object(client, "search_by_cad", return_value=mock_search_result), patch.object(client, "get_features_in_bbox_grid", return_value=[]) as mock_grid, patch.object(client, "get_features_in_bbox", return_value=[]) as mock_legacy, ): client.search_by_quarter( "66:41:0303161", include_zouit=False, include_risks=False, include_opportunity=False, ) # territorial_zones, red_lines, engineering_structures — все три должны # быть запрошены через grid-walk called_layer_ids = [call.args[0] for call in mock_grid.call_args_list] from app.services.scrapers.nspd_client import LAYERS assert ( LAYERS["territorial_zones"] in called_layer_ids ), "territorial_zones должен использовать grid-walk" assert LAYERS["red_lines"] in called_layer_ids, "red_lines должен использовать grid-walk" assert ( LAYERS["engineering_structures"] in called_layer_ids ), "engineering_structures должен использовать grid-walk" # parcels и buildings — legacy, не grid called_legacy_ids = [call.args[0] for call in mock_legacy.call_args_list] assert LAYERS["parcels"] in called_legacy_ids, "parcels должен идти через legacy" assert LAYERS["buildings"] in called_legacy_ids, "buildings должен идти через legacy" def test_fetch_layer_uses_legacy_for_egrn_layers(self) -> None: """parcels и buildings → вызывают legacy get_features_in_bbox (не grid).""" client = NSPDClient() mock_search_result = self._make_search_result() with ( patch.object(client, "search_by_cad", return_value=mock_search_result), patch.object(client, "get_features_in_bbox_grid", return_value=[]) as mock_grid, patch.object(client, "get_features_in_bbox", return_value=[]) as mock_legacy, ): client.search_by_quarter( "66:41:0303161", include_zouit=False, include_risks=False, include_opportunity=False, ) from app.services.scrapers.nspd_client import LAYERS called_legacy_ids = [call.args[0] for call in mock_legacy.call_args_list] called_grid_ids = [call.args[0] for call in mock_grid.call_args_list] assert LAYERS["parcels"] in called_legacy_ids assert LAYERS["buildings"] in called_legacy_ids # EGRN layers должны НЕ попасть в grid-walk assert LAYERS["parcels"] not in called_grid_ids, "parcels не должен использовать grid" assert LAYERS["buildings"] not in called_grid_ids, "buildings не должен использовать grid" def test_engineering_structures_classified_kind_enriched(self) -> None: """engineering_structures features получают classified_kind в properties.""" from app.services.scrapers.nspd_client import LAYERS # Создаём feature с mutable properties (как реальный NSPDFeature) eng_feat = MagicMock(spec=NSPDFeature) eng_feat.feature_id = "eng-001" eng_feat.geometry = None eng_feat.properties = {"params_name": "Газопровод высокого давления"} client = NSPDClient() mock_search_result = self._make_search_result() def _grid_side_effect(layer_id: int, bbox: Any, **kwargs: Any) -> list[Any]: if layer_id == LAYERS["engineering_structures"]: return [eng_feat] return [] with ( patch.object(client, "search_by_cad", return_value=mock_search_result), patch.object(client, "get_features_in_bbox_grid", side_effect=_grid_side_effect), patch.object(client, "get_features_in_bbox", return_value=[]), ): dump = client.search_by_quarter( "66:41:0303161", include_zouit=False, include_risks=False, include_opportunity=False, ) assert len(dump.engineering_structures) == 1 enriched = dump.engineering_structures[0] got = enriched.properties.get("classified_kind") assert got == "gas", f"Ожидали classified_kind='gas', получили {got!r}" # ── Integration marker (skip in CI) ────────────────────────────────────────── @pytest.mark.integration def test_live_nspd_grid_walk_skipped() -> None: """Placeholder для ручного запуска live NSPD grid-walk. Запуск: cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -m integration -s При запуске — делает реальные HTTP-запросы к nspd.gov.ru. """ pytest.skip("Live NSPD integration test — запускать вручную с -m integration")