gendesign/backend/tests/scrapers/test_nspd_grid_walk.py

"""Unit-тесты для NSPDClient.get_features_in_bbox_grid + classify_engineering_kind.

Sub-PR A foundation (#126): grid-walk — НЕ live HTTP (all mocked).

Запуск:
    cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -v

Live integration (помечены @pytest.mark.integration — пропускаются в CI):
    cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -m integration -s
"""

from __future__ import annotations

from typing import Any, ClassVar
from unittest.mock import AsyncMock, MagicMock, patch

import pytest

from app.services.scrapers.nspd_client import NSPDClient, NSPDFeature
from app.services.scrapers.nspd_denorm import classify_engineering_kind

# ── Helpers ────────────────────────────────────────────────────────────────────


def _make_bulk_feature(feature_id: str | int | None, props: dict[str, Any]) -> MagicMock:
    """Создать mock NSPDBulkFeature с нужными полями."""
    feat = MagicMock()
    feat.id = feature_id
    feat.geometry = None
    feat.properties = props
    return feat


# ── Grid-walk count tests ──────────────────────────────────────────────────────


class TestGetFeaturesInBboxGrid:
    """Тесты grid-walk метода (без live NSPD)."""

    # bbox 700×700м в EPSG:3857 (типичный квартал ЕКБ)
    BBOX: tuple[float, float, float, float] = (
        6_600_000.0,
        7_700_000.0,
        6_600_700.0,
        7_700_700.0,
    )

    def _patch_bulk_client(self, return_features: list[list[Any]]) -> tuple[Any, AsyncMock]:
        """Патч NSPDBulkClient.wms_feature_info.

        return_features: список ответов — по одному на каждый вызов wms_feature_info
        (в порядке вызовов). Если список короче числа calls — последний элемент
        повторяется.
        """
        call_idx: list[int] = [0]

        async def _side_effect(*args: Any, **kwargs: Any) -> list[Any]:
            idx = min(call_idx[0], len(return_features) - 1)
            call_idx[0] += 1
            return return_features[idx]

        mock_wms = AsyncMock(side_effect=_side_effect)
        return mock_wms, mock_wms

    def test_grid_n7_calls_49_times(self) -> None:
        """grid_n=7 → ровно 49 вызовов wms_feature_info."""
        call_count: list[int] = [0]

        async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
            call_count[0] += 1
            return []

        mock_client_instance = AsyncMock()
        mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
        mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
        mock_client_instance.__aexit__ = AsyncMock(return_value=None)

        with patch(
            "app.scrapers.nspd_bulk_client.NSPDBulkClient",
            return_value=mock_client_instance,
        ):
            client = NSPDClient()
            result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=7, step_m=1.0)

        assert call_count[0] == 49, f"Ожидали 49 вызовов, получили {call_count[0]}"
        assert result == []

    def test_grid_n3_calls_9_times(self) -> None:
        """grid_n=3 → ровно 9 вызовов."""
        call_count: list[int] = [0]

        async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
            call_count[0] += 1
            return []

        mock_client_instance = AsyncMock()
        mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
        mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
        mock_client_instance.__aexit__ = AsyncMock(return_value=None)

        with patch(
            "app.scrapers.nspd_bulk_client.NSPDBulkClient",
            return_value=mock_client_instance,
        ):
            client = NSPDClient()
            result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=3, step_m=1.0)

        assert call_count[0] == 9
        assert result == []

    def test_dedup_same_feature_id(self) -> None:
        """Одинаковые feature_id из соседних ячеек → одна запись."""
        feat_a = _make_bulk_feature("feat-001", {"cad_num": "66:41:001:1"})
        feat_b = _make_bulk_feature("feat-001", {"cad_num": "66:41:001:1"})  # дубликат
        feat_c = _make_bulk_feature("feat-002", {"cad_num": "66:41:001:2"})

        async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
            return [feat_a, feat_b, feat_c]

        mock_client_instance = AsyncMock()
        mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
        mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
        mock_client_instance.__aexit__ = AsyncMock(return_value=None)

        with patch(
            "app.scrapers.nspd_bulk_client.NSPDBulkClient",
            return_value=mock_client_instance,
        ):
            client = NSPDClient()
            result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0)

        # 4 cells × 3 features = 12 raw, но feature_id уникальных 2
        feature_ids = [f.feature_id for f in result]
        assert "feat-001" in feature_ids
        assert "feat-002" in feature_ids
        # нет дубликатов
        assert len(feature_ids) == len(set(feature_ids))

    def test_dedup_by_cad_num_when_no_feature_id(self) -> None:
        """Если feature_id=None — дедупликация по cad_num."""
        feat_a = _make_bulk_feature(None, {"cad_num": "66:41:0000001:100"})
        feat_b = _make_bulk_feature(None, {"cad_num": "66:41:0000001:100"})  # дубликат

        call_n: list[int] = [0]

        async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
            call_n[0] += 1
            # Первая ячейка возвращает feat_a, вторая — feat_b
            return [feat_a] if call_n[0] == 1 else [feat_b]

        mock_client_instance = AsyncMock()
        mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
        mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
        mock_client_instance.__aexit__ = AsyncMock(return_value=None)

        with patch(
            "app.scrapers.nspd_bulk_client.NSPDBulkClient",
            return_value=mock_client_instance,
        ):
            client = NSPDClient()
            result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0)

        cad_nums = [f.properties.get("cad_num") for f in result]
        assert cad_nums.count("66:41:0000001:100") == 1, "Дубликат не дедуплицирован"

    def test_error_in_one_cell_does_not_abort(self) -> None:
        """Ошибка в одной ячейке не останавливает весь grid-walk."""
        good_feat = _make_bulk_feature("feat-ok", {"cad_num": "66:41:001:1"})
        call_n: list[int] = [0]

        async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
            call_n[0] += 1
            if call_n[0] == 1:
                raise RuntimeError("Simulated cell error")
            return [good_feat]

        mock_client_instance = AsyncMock()
        mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
        mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
        mock_client_instance.__aexit__ = AsyncMock(return_value=None)

        with patch(
            "app.scrapers.nspd_bulk_client.NSPDBulkClient",
            return_value=mock_client_instance,
        ):
            client = NSPDClient()
            # Не должно бросать исключение
            result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0)

        # 4 cells: 1 error + 3 good_feat → 1 unique feature
        assert any(f.feature_id == "feat-ok" for f in result)

    # ── #2464-G: отказ слоя больше не маскируется пустым результатом ──────────

    def _grid(self, side_effect: Any, *, grid_n: int = 2) -> list[NSPDFeature]:
        """Прогнать grid-walk с подменённым wms_feature_info."""
        mock_client_instance = AsyncMock()
        mock_client_instance.wms_feature_info = AsyncMock(side_effect=side_effect)
        mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
        mock_client_instance.__aexit__ = AsyncMock(return_value=None)
        with patch(
            "app.scrapers.nspd_bulk_client.NSPDBulkClient",
            return_value=mock_client_instance,
        ):
            return NSPDClient().get_features_in_bbox_grid(
                36328, self.BBOX, grid_n=grid_n, step_m=1.0
            )

    def test_waf_403_aborts_grid_instead_of_empty_result(self) -> None:
        """403 WAF на ячейке — бан IP, обход прерывается.

        До #2464-G исключение глушилось и метод отдавал [] — «зон здесь нет»,
        неотличимое от честного пустого слоя. На проде это 124 дампа из 669
        с territorial_zones_count=0, у 50 из которых соседний legacy-слой
        данные всё-таки нашёл.
        """
        from app.scrapers.nspd_bulk_client import NspdBulkWafError

        async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
            raise NspdBulkWafError("HTTP 403 WAF")

        with pytest.raises(NspdBulkWafError):
            self._grid(_wms)

    def test_all_cells_5xx_raises_instead_of_empty_result(self) -> None:
        """Все ячейки упали с 5xx — слой лёг целиком, а не «пуст»."""
        from app.scrapers.nspd_bulk_client import NspdBulkServerError

        async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
            raise NspdBulkServerError("HTTP 500 ServiceException")

        with pytest.raises(NspdBulkServerError):
            self._grid(_wms)

    def test_partial_5xx_keeps_data_and_does_not_raise(self) -> None:
        """Часть ячеек 5xx, часть прошла — отдаём собранное, не бросаем.

        Контроль к двум тестам выше: правка НЕ превращает любую ошибку в отказ.
        Именно этот тест ловил бы обратную крайность — «чуть что, роняем обход».
        """
        from app.scrapers.nspd_bulk_client import NspdBulkServerError

        good_feat = _make_bulk_feature("feat-ok", {"cad_num": "66:41:001:1"})
        call_n: list[int] = [0]

        async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
            call_n[0] += 1
            if call_n[0] <= 2:
                raise NspdBulkServerError("HTTP 500 ServiceException")
            return [good_feat]

        result = self._grid(_wms)
        assert any(f.feature_id == "feat-ok" for f in result), (
            "успешные ячейки должны попасть в результат, даже если часть слоя упала"
        )

    def test_returns_nspd_feature_instances(self) -> None:
        """Метод возвращает list[NSPDFeature] а не NSPDBulkFeature."""
        bulk_feat = _make_bulk_feature("feat-xyz", {"cad_num": "66:41:001:1"})

        async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
            return [bulk_feat]

        mock_client_instance = AsyncMock()
        mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
        mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
        mock_client_instance.__aexit__ = AsyncMock(return_value=None)

        with patch(
            "app.scrapers.nspd_bulk_client.NSPDBulkClient",
            return_value=mock_client_instance,
        ):
            client = NSPDClient()
            result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=1, step_m=1.0)

        assert all(isinstance(f, NSPDFeature) for f in result)

    def test_auto_reduce_grid_for_small_bbox(self) -> None:
        """Маленький bbox + большой grid_n + большой step_m → grid_n уменьшается."""
        # bbox 60×60м, step_m=50, grid_n=7 → effective_n=min(7, int(60/50), int(60/50)) = 1
        small_bbox: tuple[float, float, float, float] = (
            6_600_000.0,
            7_700_000.0,
            6_600_060.0,
            7_700_060.0,
        )
        call_count: list[int] = [0]

        async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
            call_count[0] += 1
            return []

        mock_client_instance = AsyncMock()
        mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
        mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
        mock_client_instance.__aexit__ = AsyncMock(return_value=None)

        with patch(
            "app.scrapers.nspd_bulk_client.NSPDBulkClient",
            return_value=mock_client_instance,
        ):
            client = NSPDClient()
            client.get_features_in_bbox_grid(36328, small_bbox, grid_n=7, step_m=50.0)

        # effective_n = 1 → 1 ячейка
        assert call_count[0] == 1


# ── Classifier tests ───────────────────────────────────────────────────────────


class TestClassifyEngineeringKind:
    """Smoke-тесты классификатора инженерных сооружений."""

    @pytest.mark.parametrize(
        "props, expected",
        [
            # gas
            ({"params_name": "Газопровод высокого давления"}, "gas"),
            ({"name": "Газопровод ПЭ SDR11 160мм"}, "gas"),
            ({"params_purpose": "Газоснабжение"}, "gas"),
            ({"name": "ГРП №12"}, "gas"),
            # electric
            ({"name": "КЛ 10 кВ ТП 64102"}, "electric"),
            ({"params_name": "ВЛ-10кВ Ф-14"}, "electric"),
            ({"name": "ТП 1234"}, "electric"),
            ({"params_purpose": "Электроэнергетика и связь"}, "electric"),
            ({"name": "Подстанция 110/10 кВ"}, "electric"),
            # water
            ({"params_purpose": "Водопровод хозбытовой"}, "water"),
            ({"name": "Водовод Ду300"}, "water"),
            ({"params_name": "Сеть водоснабжения"}, "water"),
            # heat
            ({"params_name": "Тепловая сеть"}, "heat"),
            ({"name": "Теплосеть квартал 24"}, "heat"),
            ({"params_purpose": "Теплоснабжение жилых домов"}, "heat"),
            ({"name": "ТЭЦ-4 отпайка"}, "heat"),
            # sewage
            ({"name": "Канализация"}, "sewage"),
            ({"params_name": "Сеть канализации Ду200"}, "sewage"),
            ({"name": "Ливневая канализация"}, "sewage"),
            # other
            ({"name": "Объект не классифицирован"}, "other"),
            ({}, "other"),
            ({"params_purpose": None}, "other"),
        ],
    )
    def test_classify(self, props: dict[str, Any], expected: str) -> None:
        assert classify_engineering_kind(props) == expected, (
            f"props={props!r} → expected {expected!r}"
        )

    def test_field_priority_params_name_over_purpose(self) -> None:
        """params_name проверяется раньше purpose."""
        props = {
            "params_name": "Газопровод",  # → gas
            "params_purpose": "Водоснабжение",  # → water (если бы проверялось первым)
        }
        # gas-паттерн найдётся в combined строке первым по порядку _ENGINEERING_PATTERNS
        assert classify_engineering_kind(props) == "gas"

    def test_case_insensitive(self) -> None:
        """Паттерны case-insensitive."""
        assert classify_engineering_kind({"name": "ГАЗОПРОВОД ВЫСОКОГО ДАВЛЕНИЯ"}) == "gas"
        assert classify_engineering_kind({"name": "канализация бытовая"}) == "sewage"


# ── Sub-PR B: _fetch_layer dispatch tests ─────────────────────────────────────


class TestFetchLayerDispatch:
    """Проверяем что _fetch_layer внутри search_by_quarter правильно диспатчит
    area-слои на get_features_in_bbox_grid, а EGRN-слои — на legacy get_features_in_bbox.

    Моки: search_by_cad и get_features_in_bbox_grid / get_features_in_bbox
    через patch, без live HTTP.
    """

    # Минимальный bbox квартала (Web Mercator), который возвращает search_by_cad
    _MOCK_GEOM: ClassVar[dict[str, Any]] = {
        "type": "Polygon",
        "coordinates": [
            [
                [6_600_000.0, 7_700_000.0],
                [6_600_700.0, 7_700_000.0],
                [6_600_700.0, 7_700_700.0],
                [6_600_000.0, 7_700_700.0],
                [6_600_000.0, 7_700_000.0],
            ]
        ],
    }

    def _make_search_result(self) -> MagicMock:
        """Mock NSPDSearchResult с первым feature, у которого есть geometry."""
        feat = MagicMock(spec=NSPDFeature)
        feat.geometry = self._MOCK_GEOM
        feat.properties = {}
        feat.feature_id = "q-001"
        result = MagicMock()
        result.first = feat
        return result

    def test_fetch_layer_uses_grid_walk_for_area_layers(self) -> None:
        """territorial_zones (area layer) → вызывает get_features_in_bbox_grid."""
        client = NSPDClient()
        mock_search_result = self._make_search_result()

        with (
            patch.object(client, "search_by_cad", return_value=mock_search_result),
            patch.object(client, "get_features_in_bbox_grid", return_value=[]) as mock_grid,
            patch.object(client, "get_features_in_bbox", return_value=[]) as mock_legacy,
        ):
            client.search_by_quarter(
                "66:41:0303161",
                include_zouit=False,
                include_risks=False,
                include_opportunity=False,
            )

        # territorial_zones, red_lines, engineering_structures — все три должны
        # быть запрошены через grid-walk
        called_layer_ids = [call.args[0] for call in mock_grid.call_args_list]
        from app.services.scrapers.nspd_client import LAYERS

        assert LAYERS["territorial_zones"] in called_layer_ids, (
            "territorial_zones должен использовать grid-walk"
        )
        assert LAYERS["red_lines"] in called_layer_ids, "red_lines должен использовать grid-walk"
        assert LAYERS["engineering_structures"] in called_layer_ids, (
            "engineering_structures должен использовать grid-walk"
        )
        # parcels и buildings — legacy, не grid
        called_legacy_ids = [call.args[0] for call in mock_legacy.call_args_list]
        assert LAYERS["parcels"] in called_legacy_ids, "parcels должен идти через legacy"
        assert LAYERS["buildings"] in called_legacy_ids, "buildings должен идти через legacy"

    def test_fetch_layer_uses_legacy_for_egrn_layers(self) -> None:
        """parcels и buildings → вызывают legacy get_features_in_bbox (не grid)."""
        client = NSPDClient()
        mock_search_result = self._make_search_result()

        with (
            patch.object(client, "search_by_cad", return_value=mock_search_result),
            patch.object(client, "get_features_in_bbox_grid", return_value=[]) as mock_grid,
            patch.object(client, "get_features_in_bbox", return_value=[]) as mock_legacy,
        ):
            client.search_by_quarter(
                "66:41:0303161",
                include_zouit=False,
                include_risks=False,
                include_opportunity=False,
            )

        from app.services.scrapers.nspd_client import LAYERS

        called_legacy_ids = [call.args[0] for call in mock_legacy.call_args_list]
        called_grid_ids = [call.args[0] for call in mock_grid.call_args_list]

        assert LAYERS["parcels"] in called_legacy_ids
        assert LAYERS["buildings"] in called_legacy_ids
        # EGRN layers должны НЕ попасть в grid-walk
        assert LAYERS["parcels"] not in called_grid_ids, "parcels не должен использовать grid"
        assert LAYERS["buildings"] not in called_grid_ids, "buildings не должен использовать grid"

    def test_engineering_structures_classified_kind_enriched(self) -> None:
        """engineering_structures features получают classified_kind в properties."""
        from app.services.scrapers.nspd_client import LAYERS

        # Создаём feature с mutable properties (как реальный NSPDFeature)
        eng_feat = MagicMock(spec=NSPDFeature)
        eng_feat.feature_id = "eng-001"
        eng_feat.geometry = None
        eng_feat.properties = {"params_name": "Газопровод высокого давления"}

        client = NSPDClient()
        mock_search_result = self._make_search_result()

        def _grid_side_effect(layer_id: int, bbox: Any, **kwargs: Any) -> list[Any]:
            if layer_id == LAYERS["engineering_structures"]:
                return [eng_feat]
            return []

        with (
            patch.object(client, "search_by_cad", return_value=mock_search_result),
            patch.object(client, "get_features_in_bbox_grid", side_effect=_grid_side_effect),
            patch.object(client, "get_features_in_bbox", return_value=[]),
        ):
            dump = client.search_by_quarter(
                "66:41:0303161",
                include_zouit=False,
                include_risks=False,
                include_opportunity=False,
            )

        assert len(dump.engineering_structures) == 1
        enriched = dump.engineering_structures[0]
        got = enriched.properties.get("classified_kind")
        assert got == "gas", f"Ожидали classified_kind='gas', получили {got!r}"


# ── Integration marker (skip in CI) ──────────────────────────────────────────


@pytest.mark.integration
def test_live_nspd_grid_walk_skipped() -> None:
    """Placeholder для ручного запуска live NSPD grid-walk.

    Запуск:
        cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -m integration -s

    При запуске — делает реальные HTTP-запросы к nspd.gov.ru.
    """
    pytest.skip("Live NSPD integration test — запускать вручную с -m integration")