gendesign/backend/tests/scrapers/test_nspd_bulk_client.py

"""Тесты для NSPDBulkClient (nspd_bulk_client.py) — PR 2/5 issue #168.

Структура:
  - Unit тесты (без сети) — параметры, семафор, parse-логика
  - E2E тесты (@pytest.mark.slow + @pytest.mark.skip по умолчанию) —
    реальные запросы к NSPD, CI не дёргает

Запуск e2e вручную:
    cd backend && uv run pytest tests/scrapers/test_nspd_bulk_client.py -m slow -s -v
"""

from __future__ import annotations

import asyncio
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch

import pytest

from app.schemas.nspd_bulk import NSPDBulkFeature, ObjectsListing, QuarterSnapshot
from app.scrapers.nspd_bulk_client import (
    NSPDBulkClient,
    NspdBulkError,
    NspdBulkWafError,
)

# ── Fixtures ──────────────────────────────────────────────────────────────────


@pytest.fixture
def sample_quarter_response() -> dict[str, Any]:
    """Минимальный валидный ответ search_by_quarter из NSPD.

    Real NSPD shape (verified live on 66:41:0303161 in #177):
    - `properties.category` (НЕ categoryId)
    - `meta` лежит на top-level (не в data.meta)
    """
    return {
        "data": {
            "type": "FeatureCollection",
            "features": [
                {
                    "id": 1001,
                    "type": "Feature",
                    "geometry": {
                        "type": "Polygon",
                        "coordinates": [
                            [
                                [6700000.0, 7700000.0],
                                [6701000.0, 7700000.0],
                                [6701000.0, 7701000.0],
                                [6700000.0, 7701000.0],
                                [6700000.0, 7700000.0],
                            ],
                        ],
                    },
                    "properties": {
                        "category": 36368,
                        "categoryName": "Земельные участки ЕГРН",
                        "options": {
                            "cad_num": "66:41:0303161:123",
                            "area": 500.0,
                        },
                    },
                },
                {
                    "id": 2002,
                    "type": "Feature",
                    "geometry": {"type": "Point", "coordinates": [6700500.0, 7700500.0]},
                    "properties": {
                        "category": 36369,
                        "categoryName": "Здания",
                        "options": {
                            "cad_num": "66:41:0303161:456:1",
                            "floors": "5",
                        },
                    },
                },
            ],
        },
        # meta — top-level, primary path в search_by_quarter
        "meta": [
            {"categoryId": 36368, "totalCount": 170},
            {"categoryId": 36369, "totalCount": 256},
            {"categoryId": 36383, "totalCount": 75},
            {"categoryId": 37163, "totalCount": 5877},
            {"categoryId": 39663, "totalCount": 1},
        ],
    }


@pytest.fixture
def sample_wms_response() -> dict[str, Any]:
    """Минимальный WMS GetFeatureInfo response."""
    return {
        "type": "FeatureCollection",
        "features": [
            {
                "id": 3003,
                "type": "Feature",
                "geometry": {
                    "type": "Polygon",
                    "coordinates": [
                        [
                            [6700100.0, 7700100.0],
                            [6700900.0, 7700100.0],
                            [6700900.0, 7700900.0],
                            [6700100.0, 7700900.0],
                            [6700100.0, 7700100.0],
                        ],
                    ],
                },
                "properties": {
                    "category": 36368,
                    "options": {"cad_num": "66:41:0303161:789"},
                },
            }
        ],
    }


@pytest.fixture
def sample_tab_group_response() -> dict[str, Any]:
    """Минимальный tab-group-data response для здания."""
    return {
        "title": "Список объектов",
        "object": [
            {"title": "Помещения (количество)", "value": ["264"]},
            {"title": "Машино-места (количество)", "value": ["4"]},
            {
                "title": "Помещения (список)",
                "value": ["66:41:0303161:100:1", "66:41:0303161:100:2"],
            },
            {
                "title": "Машино-места (список)",
                "value": ["66:41:0303161:100:265", "66:41:0303161:100:266"],
            },
        ],
    }


# ── Unit tests: schemas ───────────────────────────────────────────────────────


def test_nspd_bulk_feature_parse_basic() -> None:
    """NSPDBulkFeature парсит id, geometry, properties (primary path: properties.category)."""
    raw = {
        "id": 42,
        "type": "Feature",
        "geometry": {"type": "Point", "coordinates": [6700000.0, 7700000.0]},
        "properties": {
            "category": 36368,
            "options": {"cad_num": "66:41:0303161:1", "area": 300.0},
        },
    }
    feat = NSPDBulkFeature.model_validate(raw)
    assert feat.id == 42
    assert feat.category_id == 36368
    assert feat.cad_num == "66:41:0303161:1"
    assert feat.options.area == 300.0


def test_nspd_bulk_feature_extra_fields_preserved() -> None:
    """extra='allow' → неизвестные поля сохраняются."""
    raw = {
        "id": 1,
        "geometry": None,
        "properties": {},
        "unknown_future_field": "value",
    }
    feat = NSPDBulkFeature.model_validate(raw)
    assert feat.model_extra["unknown_future_field"] == "value"  # type: ignore[index]


def test_nspd_options_effective_cad_num_prefers_cad_num() -> None:
    """effective_cad_num берёт cad_num если оба заданы."""
    from app.schemas.nspd_bulk import NSPDOptions

    opts = NSPDOptions(cad_num="66:41:0303161:1", cad_number="66:41:0303161:2")
    assert opts.effective_cad_num == "66:41:0303161:1"


def test_nspd_options_fallback_to_cad_number() -> None:
    """effective_cad_num fallback → cad_number если cad_num is None."""
    from app.schemas.nspd_bulk import NSPDOptions

    opts = NSPDOptions(cad_number="66:41:0303161:2")
    assert opts.effective_cad_num == "66:41:0303161:2"


def test_quarter_snapshot_overflow_categories() -> None:
    """overflow_categories = категории с totalCount > 20."""
    snap = QuarterSnapshot(
        quarter="66:41:0303161",
        fetched_at="2026-05-15T00:00:00+00:00",
        meta_counts={36368: 170, 36369: 256, 36383: 5, 39663: 1},
    )
    overflow = snap.overflow_categories
    assert 36368 in overflow
    assert 36369 in overflow
    assert 36383 not in overflow  # 5 <= 20
    assert 39663 not in overflow  # 1 <= 20


def test_objects_listing_defaults() -> None:
    """ObjectsListing дефолтные значения."""
    listing = ObjectsListing(objdoc_id=12345)
    assert listing.flats_count == 0
    assert listing.parking_count == 0
    assert listing.flats_cad_nums == []


def test_nspd_options_parses_camelcase_objdoc_id() -> None:
    """#96: NSPD search отдаёт objdocId/registersId camelCase → AliasChoices ловит.

    Regression: snake-only парсинг давал None (verified live на
    66:41:0106036:183, objdocId=40995027) и ломал list_objects_in_building.
    """
    from app.schemas.nspd_bulk import NSPDOptions

    opts = NSPDOptions.model_validate({"objdocId": 40995027, "registersId": 36441})
    assert opts.objdoc_id == 40995027
    assert opts.registers_id == 36441


def test_nspd_options_snake_case_objdoc_still_works() -> None:
    """#96: bulk grid-walk snake_case objdoc_id по-прежнему парсится (back-compat)."""
    from app.schemas.nspd_bulk import NSPDOptions

    opts = NSPDOptions.model_validate({"objdoc_id": 777, "registers_id": 36441})
    assert opts.objdoc_id == 777
    assert opts.registers_id == 36441


def test_objects_listing_parking_ratio() -> None:
    """#96: parking_ratio = машино-места / помещения, округление 0.01."""
    listing = ObjectsListing(objdoc_id=1, flats_count=200, parking_count=240)
    assert listing.parking_ratio == 1.2


def test_objects_listing_parking_ratio_none_when_no_flats() -> None:
    """#96: parking_ratio = None при flats_count=0 (нельзя делить, не выдумываем 0)."""
    listing = ObjectsListing(objdoc_id=1, flats_count=0, parking_count=0)
    assert listing.parking_ratio is None


def test_objects_listing_parking_ratio_zero_parking() -> None:
    """#96: 0 машино-мест при N помещений → ratio 0.0 (реальный дефицит, не None)."""
    listing = ObjectsListing(objdoc_id=1, flats_count=232, parking_count=0)
    assert listing.parking_ratio == 0.0


# ── Unit tests: client parse logic (mocked HTTP) ─────────────────────────────


@pytest.mark.asyncio
async def test_search_by_quarter_parses_features(
    sample_quarter_response: dict[str, Any],
) -> None:
    """search_by_quarter корректно парсит features и meta_counts."""
    async with NSPDBulkClient() as client:
        with patch.object(client, "_get_json", new=AsyncMock(return_value=sample_quarter_response)):
            snap = await client.search_by_quarter("66:41:0303161")

    assert snap.quarter == "66:41:0303161"
    assert len(snap.features) == 2
    assert snap.meta_counts[36368] == 170
    assert snap.meta_counts[36369] == 256
    assert snap.meta_counts[37163] == 5877
    # overflow: категории с >20
    overflow = snap.overflow_categories
    assert 36368 in overflow
    assert 36369 in overflow
    assert 36383 in overflow
    assert 37163 in overflow
    assert 39663 not in overflow  # totalCount=1 → no overflow


@pytest.mark.asyncio
async def test_search_by_quarter_empty_on_404() -> None:
    """search_by_quarter возвращает пустой snapshot при 404."""
    async with NSPDBulkClient() as client:
        with patch.object(
            client,
            "_get_json",
            new=AsyncMock(side_effect=NspdBulkError("HTTP 404: not found")),
        ):
            snap = await client.search_by_quarter("66:41:9999999")

    assert snap.features == []
    assert snap.meta_counts == {}
    assert snap.quarter == "66:41:9999999"


@pytest.mark.asyncio
async def test_search_by_quarter_raises_on_waf() -> None:
    """search_by_quarter пробрасывает NspdBulkWafError (403) без retry."""
    async with NSPDBulkClient() as client:
        with patch.object(
            client,
            "_get_json",
            new=AsyncMock(side_effect=NspdBulkWafError("HTTP 403 WAF")),
        ):
            with pytest.raises(NspdBulkWafError):
                await client.search_by_quarter("66:41:0303161")


@pytest.mark.asyncio
async def test_wms_feature_info_parses_features(
    sample_wms_response: dict[str, Any],
) -> None:
    """wms_feature_info парсит features из GeoJSON response."""
    async with NSPDBulkClient() as client:
        with patch.object(client, "_get_json", new=AsyncMock(return_value=sample_wms_response)):
            features = await client.wms_feature_info(
                layer_id=36368,
                bbox=(6700000.0, 7700000.0, 6701000.0, 7701000.0),
                click_xy=(256, 256),
            )

    assert len(features) == 1
    assert features[0].category_id == 36368
    assert features[0].cad_num == "66:41:0303161:789"


@pytest.mark.asyncio
async def test_wms_feature_info_empty_response() -> None:
    """wms_feature_info возвращает [] при пустом response."""
    async with NSPDBulkClient() as client:
        with patch.object(client, "_get_json", new=AsyncMock(return_value={"features": []})):
            features = await client.wms_feature_info(
                layer_id=36368,
                bbox=(6700000.0, 7700000.0, 6701000.0, 7701000.0),
                click_xy=(256, 256),
            )
    assert features == []


@pytest.mark.asyncio
async def test_list_objects_in_building_parses(
    sample_tab_group_response: dict[str, Any],
) -> None:
    """list_objects_in_building парсит counts и cad_nums."""
    async with NSPDBulkClient() as client:
        with patch.object(
            client, "_get_json", new=AsyncMock(return_value=sample_tab_group_response)
        ):
            listing = await client.list_objects_in_building(objdoc_id=42065602)

    assert listing.objdoc_id == 42065602
    assert listing.flats_count == 264
    assert listing.parking_count == 4
    assert "66:41:0303161:100:1" in listing.flats_cad_nums
    assert "66:41:0303161:100:265" in listing.parking_cad_nums


@pytest.mark.asyncio
async def test_rate_limit_semaphore_max_3_concurrent() -> None:
    """Не более 3 одновременных запросов через per-instance self._sem (PR #260).

    Мокируем httpx.AsyncClient.get (нижний уровень) с задержкой, чтобы
    реальный семафор работал. Считаем max in-flight внутри семафора.
    """
    import httpx

    import app.scrapers.nspd_bulk_client as bulk_mod

    max_concurrent = 0
    current = 0
    lock = asyncio.Lock()
    call_count = 0

    async def slow_get(*args: Any, **kwargs: Any) -> httpx.Response:
        nonlocal max_concurrent, current, call_count
        # Фиксируем вход — уже внутри семафора (httpx.get вызывается после async with self._sem)
        async with lock:
            current += 1
            call_count += 1
            if current > max_concurrent:
                max_concurrent = current
        await asyncio.sleep(0.02)  # симулируем I/O задержку
        async with lock:
            current -= 1
        return httpx.Response(
            200,
            json={"data": {"features": [], "meta": []}},
            request=MagicMock(),
        )

    async with NSPDBulkClient() as client:
        assert client._client is not None
        with patch.object(client._client, "get", new=AsyncMock(side_effect=slow_get)):
            tasks = [client.search_by_quarter(f"66:41:{i:07d}") for i in range(6)]
            await asyncio.gather(*tasks)

    # self._sem(3) → не более 3 одновременно внутри slow_get
    assert max_concurrent <= 3, f"Expected ≤3 concurrent, got {max_concurrent}"
    assert call_count == 6  # все 6 вызовов прошли
    # NB: per-instance self._sem cleanup проверяется через max_concurrent <= 3 выше.
    # Module-level _SEMAPHORE удалён в PR #260 (cross-loop binding fix); smoke на capacity:
    assert bulk_mod._SEMAPHORE_LIMIT == 3


@pytest.mark.asyncio
async def test_http_403_raises_waf_no_retry() -> None:
    """403 → NspdBulkWafError немедленно, без retry."""
    import httpx

    call_count = 0

    async def fake_get(*args: Any, **kwargs: Any) -> httpx.Response:
        nonlocal call_count
        call_count += 1
        return httpx.Response(403, text="Forbidden", request=MagicMock())

    async with NSPDBulkClient() as client:
        assert client._client is not None
        with patch.object(client._client, "get", new=AsyncMock(side_effect=fake_get)):
            with pytest.raises(NspdBulkWafError):
                await client._get_json("https://nspd.gov.ru/test")

    assert call_count == 1  # ровно 1 попытка, нет retry


@pytest.mark.asyncio
async def test_http_429_backoff_exhausted() -> None:
    """429 → exponential backoff 3 retries → NspdBulkRateLimitError."""
    import httpx

    from app.scrapers.nspd_bulk_client import NspdBulkRateLimitError

    call_count = 0

    async def fake_get(*args: Any, **kwargs: Any) -> httpx.Response:
        nonlocal call_count
        call_count += 1
        return httpx.Response(429, text="Too Many Requests", request=MagicMock())

    # Патчим sleep чтобы тест не ждал 2+4+8 секунд
    async with NSPDBulkClient(max_retries=3) as client:
        assert client._client is not None
        with (
            patch.object(client._client, "get", new=AsyncMock(side_effect=fake_get)),
            patch("app.scrapers.nspd_bulk_client.asyncio.sleep", new=AsyncMock()),
        ):
            with pytest.raises(NspdBulkRateLimitError):
                await client._get_json("https://nspd.gov.ru/test")

    # 1 оригинальный + 3 retry = 4 вызова
    assert call_count == 4


# ── Issue #252: 5xx / WMS ServiceException → NspdBulkServerError ──────────────


@pytest.mark.asyncio
async def test_http_500_raises_server_error() -> None:
    """500 → NspdBulkServerError (подкласс NspdBulkError) — для retry + skip."""
    import httpx

    from app.scrapers.nspd_bulk_client import NspdBulkError, NspdBulkServerError

    async def fake_get(*args: Any, **kwargs: Any) -> httpx.Response:
        return httpx.Response(500, text="Internal Server Error", request=MagicMock())

    async with NSPDBulkClient() as client:
        assert client._client is not None
        with patch.object(client._client, "get", new=AsyncMock(side_effect=fake_get)):
            with pytest.raises(NspdBulkServerError) as exc_info:
                await client._get_json("https://nspd.gov.ru/test")

    # Подкласс NspdBulkError → autoretry_for=(Exception,) его поймает, а
    # except NspdBulkError-ветки (404/400) тоже сработают если понадобится.
    assert isinstance(exc_info.value, NspdBulkError)


@pytest.mark.asyncio
async def test_wms_service_exception_in_200_body_raises_server_error() -> None:
    """HTTP 200 с XML <ServiceException> в теле → NspdBulkServerError (не JSONDecodeError)."""
    import httpx

    from app.scrapers.nspd_bulk_client import NspdBulkServerError

    xml_body = (
        '<?xml version="1.0"?><ServiceExceptionReport>'
        "<ServiceException>Rendering error</ServiceException></ServiceExceptionReport>"
    )

    async def fake_get(*args: Any, **kwargs: Any) -> httpx.Response:
        return httpx.Response(
            200, text=xml_body, headers={"content-type": "application/xml"}, request=MagicMock()
        )

    async with NSPDBulkClient() as client:
        assert client._client is not None
        with patch.object(client._client, "get", new=AsyncMock(side_effect=fake_get)):
            with pytest.raises(NspdBulkServerError):
                await client._get_json("https://nspd.gov.ru/wms")


@pytest.mark.asyncio
async def test_wms_feature_info_propagates_server_error(
    sample_wms_response: dict[str, Any],
) -> None:
    """wms_feature_info пробрасывает NspdBulkServerError (caller считает fail-rate)."""
    from app.scrapers.nspd_bulk_client import NspdBulkServerError

    async with NSPDBulkClient() as client:
        with patch.object(
            client,
            "_get_json",
            new=AsyncMock(side_effect=NspdBulkServerError("HTTP 500 server error")),
        ):
            with pytest.raises(NspdBulkServerError):
                await client.wms_feature_info(
                    layer_id=36368,
                    bbox=(6700000.0, 7700000.0, 6701000.0, 7701000.0),
                    click_xy=(256, 256),
                )


# ── E2E tests (network, skipped by default) ───────────────────────────────────

pytestmark_slow = pytest.mark.slow


@pytest.mark.slow
@pytest.mark.skip(reason="E2E: требует сеть NSPD — запускать вручную с -m slow")
@pytest.mark.asyncio
async def test_search_by_quarter_real_ekb() -> None:
    """E2E: поиск по реальному кварталу ЕКБ — 66:41:0303161.

    Проверяет:
    - features > 100 (квартал непустой)
    - meta содержит ключи 36368/36369/36383 с totalCount > 0
    - overflow_categories не пуст (значит totalCount > 20 для хотя бы одной)
    - headers корректны (нет 403/429)
    """
    async with NSPDBulkClient() as client:
        snap = await client.search_by_quarter("66:41:0303161")

    assert len(snap.features) > 0, "Ожидали хотя бы 1 feature"
    # Meta должна содержать основные категории
    assert 36368 in snap.meta_counts, "Нет categoryId 36368 (ЗУ ЕГРН) в meta"
    assert 36369 in snap.meta_counts, "Нет categoryId 36369 (Здания) в meta"
    assert snap.meta_counts[36368] > 0, "ЗУ ЕГРН totalCount должен быть > 0"
    assert snap.meta_counts[36369] > 0, "Здания totalCount должен быть > 0"
    # В этом квартале >170 ЗУ и >256 зданий → overflow
    assert len(snap.overflow_categories) > 0, "Ожидали overflow_categories непустой"
    # total_meta_count > 100 (квартал богатый данными)
    assert snap.total_meta_count > 100


@pytest.mark.slow
@pytest.mark.skip(reason="E2E: требует сеть NSPD — запускать вручную с -m slow")
@pytest.mark.asyncio
async def test_search_by_quarter_empty() -> None:
    """E2E: несуществующий квартал → пустой snapshot."""
    async with NSPDBulkClient() as client:
        snap = await client.search_by_quarter("66:41:9999999")

    # Должно вернуть пустой snapshot (не raise)
    assert snap.features == []
    assert snap.quarter == "66:41:9999999"


@pytest.mark.slow
@pytest.mark.skip(reason="E2E: требует сеть NSPD — запускать вручную с -m slow")
@pytest.mark.asyncio
async def test_wms_feature_info_real() -> None:
    """E2E: WMS GetFeatureInfo реальный bbox из ЕКБ.

    Использует bbox центра квартала 66:41:0303161 в EPSG:3857.
    """
    # Центральная точка квартала 66:41:0303161 в 3857 (примерно)
    # ЕКБ центр ≈ 60.6°E, 56.84°N → 3857 ≈ (6748000, 7772000)
    center_x = 6748000.0
    center_y = 7772000.0
    half = 500.0  # 500м bbox
    bbox = (center_x - half, center_y - half, center_x + half, center_y + half)

    async with NSPDBulkClient() as client:
        features = await client.wms_feature_info(
            layer_id=36368,  # ЗУ ЕГРН
            bbox=bbox,
            click_xy=(256, 256),
            width=512,
            height=512,
        )

    # Хотя бы 1 feature в bbox центра ЕКБ квартала
    assert len(features) >= 1, f"Ожидали ≥1 feature, получили {len(features)}"


@pytest.mark.slow
@pytest.mark.skip(reason="E2E: требует сеть NSPD — запускать вручную с -m slow")
@pytest.mark.asyncio
async def test_list_objects_in_building_real() -> None:
    """E2E: реальный objdocId=42065602 → >150 flats_cad_nums."""
    async with NSPDBulkClient() as client:
        listing = await client.list_objects_in_building(objdoc_id=42065602)

    assert listing.objdoc_id == 42065602
    assert listing.flats_count > 150, f"Ожидали >150 помещений, получили {listing.flats_count}"
    assert (
        len(listing.flats_cad_nums) > 150
    ), f"Ожидали >150 cad_nums помещений, получили {len(listing.flats_cad_nums)}"