From 70c079dd50ca7baf2d2749b052288b8abb58b8e1 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Fri, 15 May 2026 12:42:18 +0300 Subject: [PATCH] feat(cadastre): NSPDBulkClient + schemas for bulk ingest (#168 PR2/5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add async NSPD client (nspd_bulk_client.py) with 3 new methods: - search_by_quarter: REST /api/geoportal/v2/search/geoportal — snapshot 20 per cat + meta totals - wms_feature_info: WMS GetFeatureInfo для grid-walk discovery - list_objects_in_building: tab-group-data (Q3 deferred — код ready) Rate limit: asyncio.Semaphore(3) + 0.05s jitter. 429 → exponential backoff 2→4→8s, max 3 retries. 403 (WAF) → raise immediately, no retry. Pydantic schemas (nspd_bulk.py): NSPDBulkFeature, QuarterSnapshot, ObjectsListing with extra='allow' to preserve unknown fields → raw_props JSONB downstream. 15 unit tests + 4 e2e (real NSPD, skipped in CI by default). --- backend/app/schemas/nspd_bulk.py | 134 +++++ backend/app/scrapers/nspd_bulk_client.py | 407 +++++++++++++++ backend/tests/scrapers/__init__.py | 0 .../tests/scrapers/test_nspd_bulk_client.py | 491 ++++++++++++++++++ 4 files changed, 1032 insertions(+) create mode 100644 backend/app/schemas/nspd_bulk.py create mode 100644 backend/app/scrapers/nspd_bulk_client.py create mode 100644 backend/tests/scrapers/__init__.py create mode 100644 backend/tests/scrapers/test_nspd_bulk_client.py diff --git a/backend/app/schemas/nspd_bulk.py b/backend/app/schemas/nspd_bulk.py new file mode 100644 index 00000000..1bef84ae --- /dev/null +++ b/backend/app/schemas/nspd_bulk.py @@ -0,0 +1,134 @@ +"""Pydantic schemas для NSPD bulk cadastre ingest (PR 2/5, issue #168). + +Отдельный модуль от legacy NSPDFeature dataclass (nspd_client.py) — +там frozen dataclass для on-demand flow, тут Pydantic для bulk pipeline +с extra='allow' чтобы не потерять неизвестные поля → raw_props JSONB. +""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, ConfigDict + + +class NSPDOptions(BaseModel): + """Опции кадастрового объекта из NSPD. + + Все поля Optional — разные categoryId возвращают разные subsets. + extra='allow' → любые незнакомые поля сохраняются (forward-compatible). + """ + + model_config = ConfigDict(extra="allow", populate_by_name=True) + + cad_num: str | None = None + cad_number: str | None = None # некоторые категории используют cad_number + quarter_cad_number: str | None = None + parent_cad_number: str | None = None + area: float | None = None + cost_value: float | None = None + cost_index: float | None = None + purpose: str | None = None + floors: str | int | None = None + year_built: str | int | None = None + readable_address: str | None = None + ownership_type: str | None = None + right_type: str | None = None + status: str | None = None + land_record_area: float | None = None + specified_area: float | None = None + declared_area: float | None = None + permitted_use_established_by_document: str | None = None + category_type: str | None = None + build_record_type: str | None = None + build_record_type_value: str | None = None + objdoc_id: int | None = None + registers_id: int | None = None + okato: str | None = None + kladr: str | None = None + + @property + def effective_cad_num(self) -> str | None: + """Нормализованный кад. номер: cad_num или cad_number (разные категории).""" + return self.cad_num or self.cad_number + + +class NSPDBulkFeature(BaseModel): + """GeoJSON Feature из NSPD bulk endpoints. + + Pydantic-версия (не dataclass) — для удобной сериализации + extra='allow'. + geometry — GeoJSON-dict в EPSG:3857 (transform на стороне Celery-таска). + """ + + model_config = ConfigDict(extra="allow", populate_by_name=True) + + id: int | str | None = None + type: str = "Feature" + geometry: dict[str, Any] | None = None + properties: dict[str, Any] = {} + + @property + def category_id(self) -> int | None: + """categoryId из properties (ключ для routing к target table).""" + val = self.properties.get("categoryId") + return int(val) if val is not None else None + + @property + def options(self) -> NSPDOptions: + """Распарсить options из properties.options если есть, иначе пустой.""" + opts = self.properties.get("options") or {} + return NSPDOptions.model_validate(opts) + + @property + def cad_num(self) -> str | None: + """Shortcut: cad_num из options (часто нужен в caller).""" + return self.options.effective_cad_num + + +class QuarterSnapshot(BaseModel): + """Результат search_by_quarter — snapshot данных для одного квартала. + + Первые 20 объектов на категорию из NSPD search endpoint. + overflow_categories — категории требующие grid-walk (totalCount > 20). + """ + + model_config = ConfigDict(extra="allow") + + quarter: str + fetched_at: str # ISO datetime string + features: list[NSPDBulkFeature] = [] + meta_counts: dict[int, int] = {} # {categoryId: totalCount} + + @property + def overflow_categories(self) -> list[int]: + """Категории где totalCount > 20 — нужен grid-walk для полноты.""" + return [cat_id for cat_id, count in self.meta_counts.items() if count > 20] + + @property + def total_meta_count(self) -> int: + """Суммарное количество объектов по meta (до cap).""" + return sum(self.meta_counts.values()) + + +class ObjectsListing(BaseModel): + """Результат list_objects_in_building — список помещений и машино-мест здания. + + Получается через /api/geoportal/v1/tab-group-data?tabClass=objectsList. + Q3 deferred — код готов, но в bulk_harvest_quarter MVP не используется. + """ + + model_config = ConfigDict(extra="allow") + + objdoc_id: int + flats_count: int = 0 + parking_count: int = 0 + flats_cad_nums: list[str] = [] + parking_cad_nums: list[str] = [] + + +__all__ = [ + "NSPDBulkFeature", + "NSPDOptions", + "ObjectsListing", + "QuarterSnapshot", +] diff --git a/backend/app/scrapers/nspd_bulk_client.py b/backend/app/scrapers/nspd_bulk_client.py new file mode 100644 index 00000000..f44dbeb7 --- /dev/null +++ b/backend/app/scrapers/nspd_bulk_client.py @@ -0,0 +1,407 @@ +"""NSPD bulk cadastre client — async httpx-based, rate-limited. + +PR 2/5 of GH issue #168: Bulk Cadastre Ingest (ЕКБ 2408 кварталов). + +Три новых метода: + - search_by_quarter(quarter) → QuarterSnapshot (phase 1 snapshot) + - wms_feature_info(layer_id, bbox, click_xy) → list[NSPDBulkFeature] (grid-walk) + - list_objects_in_building(objdoc_id) → ObjectsListing (Q3 deferred) + +Ключевые отличия от nspd_client.py (sync urllib): + - async httpx.AsyncClient (connection pooling, timeout objects) + - asyncio.Semaphore(3) — не более 3 одновременных запросов + - asyncio.sleep(0.05) jitter между запросами + - 429 → exponential backoff (2 → 4 → 8 с, max 3 retries) + - 403 WAF → raise немедленно (не retry, нужен manual investigate) + +Geometry: NSPD возвращает geom в EPSG:3857. ST_Transform(geom, 4326) +делается на стороне Celery-таска (PR 3) при upsert в DB. + +Rate limit: 3 req/s sustained через Semaphore(3) + 0.05s jitter. +При 429 — exponential backoff: 2s → 4s → 8s, max 3 retries. +При 403 (WAF) — raise NspdBulkWafError, не retry. +""" + +from __future__ import annotations + +import asyncio +import datetime as _dt +import logging +from typing import Any + +import httpx + +from app.schemas.nspd_bulk import NSPDBulkFeature, ObjectsListing, QuarterSnapshot + +logger = logging.getLogger(__name__) + +# ── Константы ────────────────────────────────────────────────────────────────── + +NSPD_BASE = "https://nspd.gov.ru/api" +NSPD_SEARCH_URL = f"{NSPD_BASE}/geoportal/v2/search/geoportal" +NSPD_WMS_BASE = f"{NSPD_BASE}/aeggis/v4" +NSPD_TAB_GROUP_URL = f"{NSPD_BASE}/geoportal/v1/tab-group-data" + +# Лимит одновременных запросов (глобальный, shared между всеми instance'ами) +_SEMAPHORE = asyncio.Semaphore(3) + +# Таймаут подключения и чтения +DEFAULT_TIMEOUT = httpx.Timeout(30.0, connect=10.0) + +# Headers из HAR audit 2026-05-15 +DEFAULT_HEADERS: dict[str, str] = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36" + ), + "Referer": "https://nspd.gov.ru/map", + "Accept": "*/*", + "Accept-Language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7", + "x-timezone": "Europe/Moscow", + # Origin только для POST endpoints — не включаем для GET (лишний header) +} + +# Retry settings +_MAX_RETRIES = 3 +_BACKOFF_BASE_S = 2.0 # 2 → 4 → 8 секунд + + +# ── Exceptions ───────────────────────────────────────────────────────────────── + + +class NspdBulkError(RuntimeError): + """Неуспешный ответ от NSPD bulk endpoint.""" + + +class NspdBulkWafError(NspdBulkError): + """HTTP 403 WAF блокировка — не retry, нужно manual investigate.""" + + +class NspdBulkRateLimitError(NspdBulkError): + """HTTP 429 Rate limit — исчерпаны retries после backoff.""" + + +# ── NSPDBulkClient ───────────────────────────────────────────────────────────── + + +class NSPDBulkClient: + """Async NSPD client для bulk cadastre ingest. + + Использовать как async context manager: + + async with NSPDBulkClient() as client: + snapshot = await client.search_by_quarter("66:41:0303161") + + Или с explicit lifecycle (для Celery tasks): + + client = NSPDBulkClient() + await client.__aenter__() + try: + ... + finally: + await client.__aexit__(None, None, None) + """ + + def __init__( + self, + timeout: httpx.Timeout = DEFAULT_TIMEOUT, + headers: dict[str, str] | None = None, + max_retries: int = _MAX_RETRIES, + ) -> None: + self._timeout = timeout + self._headers = headers or DEFAULT_HEADERS + self._max_retries = max_retries + self._client: httpx.AsyncClient | None = None + + async def __aenter__(self) -> NSPDBulkClient: + self._client = httpx.AsyncClient( + headers=self._headers, + timeout=self._timeout, + follow_redirects=True, + ) + return self + + async def __aexit__(self, *_: Any) -> None: + if self._client is not None: + await self._client.aclose() + self._client = None + + # ── Internal HTTP helper ─────────────────────────────────────────────────── + + async def _get_json(self, url: str, params: dict[str, Any] | None = None) -> Any: + """GET → JSON с rate-limit semaphore, jitter, retry при 429. + + Raises: + NspdBulkWafError при 403. + NspdBulkRateLimitError при 429 после исчерпания retries. + NspdBulkError при прочих 4xx/5xx. + """ + if self._client is None: + raise RuntimeError("NSPDBulkClient не инициализирован — используй async with") + + attempt = 0 + while True: + async with _SEMAPHORE: + # Небольшой jitter чтобы N одновременных запросов не начинались в 0мс + await asyncio.sleep(0.05) + try: + resp = await self._client.get(url, params=params) + except httpx.TransportError as e: + raise NspdBulkError(f"Network error: {e}") from e + + # Обработка статусов — вне семафора чтобы не держать слот при sleep + if resp.status_code == 403: + body_preview = resp.text[:200] + logger.warning("NSPD WAF 403 blocked: url=%s preview=%s", url, body_preview) + raise NspdBulkWafError(f"HTTP 403 WAF block (manual investigate required): {url}") + + if resp.status_code == 429: + attempt += 1 + if attempt > self._max_retries: + raise NspdBulkRateLimitError( + f"HTTP 429 rate-limit: exhausted {self._max_retries} retries for {url}" + ) + backoff = _BACKOFF_BASE_S * (2 ** (attempt - 1)) + logger.warning( + "NSPD 429 rate-limit: retry %d/%d after %.0fs backoff, url=%s", + attempt, + self._max_retries, + backoff, + url, + ) + await asyncio.sleep(backoff) + continue + + if resp.status_code >= 400: + body_preview = resp.text[:300] + raise NspdBulkError(f"HTTP {resp.status_code}: {url} — {body_preview}") + + return resp.json() + + # ── 1. search_by_quarter ────────────────────────────────────────────────── + + async def search_by_quarter(self, quarter: str) -> QuarterSnapshot: + """Поиск объектов в квартале через NSPD search endpoint. + + GET /api/geoportal/v2/search/geoportal?thematicSearchId=1&query={quarter} + + Возвращает первые 20 объектов на категорию + meta с totalCount. + Категории с totalCount > 20 попадают в overflow_categories + и требуют grid-walk (wms_feature_info) для полноты. + + Args: + quarter: кадастровый номер квартала в формате `66:41:NNNNNN`. + + Returns: + QuarterSnapshot с features + meta_counts. + При 404 или пустом ответе — snapshot с features=[]. + + Raises: + NspdBulkWafError при 403. + NspdBulkError при сетевых / прочих ошибках. + """ + fetched_at = _dt.datetime.now(_dt.UTC).isoformat() + + try: + data = await self._get_json( + NSPD_SEARCH_URL, + params={"thematicSearchId": 1, "query": quarter}, + ) + except NspdBulkError as e: + # 404 или пустой ответ → возвращаем пустой snapshot + err_str = str(e) + if "HTTP 404" in err_str or "HTTP 400" in err_str: + logger.info( + "search_by_quarter: quarter %s not found (404/400), returning empty", + quarter, + ) + return QuarterSnapshot(quarter=quarter, fetched_at=fetched_at) + raise + + # Парсим response + # Shape: {"data": {"type": "FeatureCollection", "features": [...], "meta": [...]}, ...} + # или: {"data": {"features": [...], "meta": [...]}} + payload = data if isinstance(data, dict) else {} + data_block = payload.get("data") or {} + if isinstance(data_block, str): + # Иногда NSPD оборачивает data в строку (Bug_Nspd_Geo_Str_Object_No_Get_Fixed) + logger.warning( + "search_by_quarter: data block is string for quarter %s, returning empty", + quarter, + ) + return QuarterSnapshot(quarter=quarter, fetched_at=fetched_at) + + raw_features: list[dict[str, Any]] = data_block.get("features") or [] + raw_meta: list[dict[str, Any]] = data_block.get("meta") or [] + + features = [NSPDBulkFeature.model_validate(f) for f in raw_features] + + # meta: [{categoryId: N, totalCount: M}, ...] + meta_counts: dict[int, int] = {} + for m in raw_meta: + cat_id = m.get("categoryId") + total = m.get("totalCount") + if cat_id is not None and total is not None: + meta_counts[int(cat_id)] = int(total) + + logger.info( + "search_by_quarter: quarter=%s features=%d meta_cats=%d overflow=%d", + quarter, + len(features), + len(meta_counts), + len([c for c, cnt in meta_counts.items() if cnt > 20]), + ) + + return QuarterSnapshot( + quarter=quarter, + fetched_at=fetched_at, + features=features, + meta_counts=meta_counts, + ) + + # ── 2. wms_feature_info ─────────────────────────────────────────────────── + + async def wms_feature_info( + self, + layer_id: int, + bbox: tuple[float, float, float, float], + click_xy: tuple[int, int], + width: int = 512, + height: int = 512, + ) -> list[NSPDBulkFeature]: + """WMS GetFeatureInfo — объекты layer'а в bbox на pixel-клике. + + GET /api/aeggis/v4/{layer_id}/wms?REQUEST=GetFeatureInfo&... + + Используется для grid-walk фазы: каждая ячейка bbox квартала + кликается в центре (click_xy = (width//2, height//2)). + + Args: + layer_id: NSPD layer ID (например 36368 для ЗУ ЕГРН). + bbox: (x_min, y_min, x_max, y_max) в EPSG:3857 (метры). + click_xy: (I, J) пиксель-координаты клика в [0..width-1, 0..height-1]. + width: ширина виртуального tile. + height: высота виртуального tile. + + Returns: + Список NSPDBulkFeature (может быть пуст). + + Raises: + NspdBulkWafError при 403. + NspdBulkError при прочих ошибках. + """ + xmin, ymin, xmax, ymax = bbox + click_i, click_j = click_xy + + url = f"{NSPD_WMS_BASE}/{layer_id}/wms" + params: dict[str, Any] = { + "REQUEST": "GetFeatureInfo", + "SERVICE": "WMS", + "VERSION": "1.3.0", + "INFO_FORMAT": "application/json", + "LAYERS": str(layer_id), + "QUERY_LAYERS": str(layer_id), + "CRS": "EPSG:3857", + "BBOX": f"{xmin},{ymin},{xmax},{ymax}", + "WIDTH": str(width), + "HEIGHT": str(height), + "I": str(click_i), + "J": str(click_j), + "STYLES": "", + "TRANSPARENT": "true", + "FORMAT": "image/png", + "FEATURE_COUNT": "10", + } + + data = await self._get_json(url, params=params) + raw_features: list[dict[str, Any]] = (data or {}).get("features") or [] + return [NSPDBulkFeature.model_validate(f) for f in raw_features] + + # ── 3. list_objects_in_building ─────────────────────────────────────────── + # Q3 deferred — метод реализован, но не вызывается в bulk_harvest_quarter MVP. + # Готов для per-building помещения/парковка фазы. + + async def list_objects_in_building( + self, + objdoc_id: int, + registers_id: int = 36441, + ) -> ObjectsListing: + """Список помещений и машино-мест в здании. + + GET /api/geoportal/v1/tab-group-data?tabClass=objectsList + &objdocId={objdoc_id}®istersId={registers_id} + + Response shape: + {title: "Список объектов", object: [ + {title: "Помещения (количество)", value: ["264"]}, + {title: "Машино-места (количество)", value: ["4"]}, + {title: "Помещения (список)", value: [...cad_nums]}, + {title: "Машино-места (список)", value: [...cad_nums]}, + ]} + + NOTE: Q3 deferred — метод готов, но НЕ вызывается в PR 2/3 MVP. + Использовать при реализации bulk_harvest_flats (отдельный task). + + Args: + objdoc_id: NSPD internal object document ID (из NSPDOptions.objdoc_id). + registers_id: NSPD registers ID (36441 — здания ЕГРН по умолчанию). + + Returns: + ObjectsListing с counts и cad_nums списками. + + Raises: + NspdBulkWafError при 403. + NspdBulkError при прочих ошибках. + """ + data = await self._get_json( + NSPD_TAB_GROUP_URL, + params={ + "tabClass": "objectsList", + "objdocId": str(objdoc_id), + "registersId": str(registers_id), + }, + ) + + flats_count = 0 + parking_count = 0 + flats_cad_nums: list[str] = [] + parking_cad_nums: list[str] = [] + + obj_list: list[dict[str, Any]] = (data or {}).get("object") or [] + for entry in obj_list: + title: str = (entry.get("title") or "").lower() + values: list[Any] = entry.get("value") or [] + + if "помещения" in title and "количество" in title: + try: + flats_count = int(values[0]) if values else 0 + except (ValueError, TypeError): + flats_count = 0 + + elif "машино-места" in title and "количество" in title: + try: + parking_count = int(values[0]) if values else 0 + except (ValueError, TypeError): + parking_count = 0 + + elif "помещения" in title and "список" in title: + flats_cad_nums = [str(v) for v in values if v] + + elif "машино-места" in title and "список" in title: + parking_cad_nums = [str(v) for v in values if v] + + return ObjectsListing( + objdoc_id=objdoc_id, + flats_count=flats_count, + parking_count=parking_count, + flats_cad_nums=flats_cad_nums, + parking_cad_nums=parking_cad_nums, + ) + + +__all__ = [ + "NSPDBulkClient", + "NspdBulkError", + "NspdBulkRateLimitError", + "NspdBulkWafError", +] diff --git a/backend/tests/scrapers/__init__.py b/backend/tests/scrapers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/tests/scrapers/test_nspd_bulk_client.py b/backend/tests/scrapers/test_nspd_bulk_client.py new file mode 100644 index 00000000..fe7a4ddb --- /dev/null +++ b/backend/tests/scrapers/test_nspd_bulk_client.py @@ -0,0 +1,491 @@ +"""Тесты для NSPDBulkClient (nspd_bulk_client.py) — PR 2/5 issue #168. + +Структура: + - Unit тесты (без сети) — параметры, семафор, parse-логика + - E2E тесты (@pytest.mark.slow + @pytest.mark.skip по умолчанию) — + реальные запросы к NSPD, CI не дёргает + +Запуск e2e вручную: + cd backend && uv run pytest tests/scrapers/test_nspd_bulk_client.py -m slow -s -v +""" + +from __future__ import annotations + +import asyncio +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from app.schemas.nspd_bulk import NSPDBulkFeature, ObjectsListing, QuarterSnapshot +from app.scrapers.nspd_bulk_client import ( + NSPDBulkClient, + NspdBulkError, + NspdBulkWafError, +) + +# ── Fixtures ────────────────────────────────────────────────────────────────── + + +@pytest.fixture +def sample_quarter_response() -> dict[str, Any]: + """Минимальный валидный ответ search_by_quarter из NSPD.""" + return { + "data": { + "type": "FeatureCollection", + "features": [ + { + "id": 1001, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [6700000.0, 7700000.0], + [6701000.0, 7700000.0], + [6701000.0, 7701000.0], + [6700000.0, 7701000.0], + [6700000.0, 7700000.0], + ], + ], + }, + "properties": { + "categoryId": 36368, + "categoryName": "Земельные участки ЕГРН", + "options": { + "cad_num": "66:41:0303161:123", + "area": 500.0, + }, + }, + }, + { + "id": 2002, + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [6700500.0, 7700500.0]}, + "properties": { + "categoryId": 36369, + "categoryName": "Здания", + "options": { + "cad_num": "66:41:0303161:456:1", + "floors": "5", + }, + }, + }, + ], + "meta": [ + {"categoryId": 36368, "totalCount": 170}, + {"categoryId": 36369, "totalCount": 256}, + {"categoryId": 36383, "totalCount": 75}, + {"categoryId": 37163, "totalCount": 5877}, + {"categoryId": 39663, "totalCount": 1}, + ], + } + } + + +@pytest.fixture +def sample_wms_response() -> dict[str, Any]: + """Минимальный WMS GetFeatureInfo response.""" + return { + "type": "FeatureCollection", + "features": [ + { + "id": 3003, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [6700100.0, 7700100.0], + [6700900.0, 7700100.0], + [6700900.0, 7700900.0], + [6700100.0, 7700900.0], + [6700100.0, 7700100.0], + ], + ], + }, + "properties": { + "categoryId": 36368, + "options": {"cad_num": "66:41:0303161:789"}, + }, + } + ], + } + + +@pytest.fixture +def sample_tab_group_response() -> dict[str, Any]: + """Минимальный tab-group-data response для здания.""" + return { + "title": "Список объектов", + "object": [ + {"title": "Помещения (количество)", "value": ["264"]}, + {"title": "Машино-места (количество)", "value": ["4"]}, + { + "title": "Помещения (список)", + "value": ["66:41:0303161:100:1", "66:41:0303161:100:2"], + }, + { + "title": "Машино-места (список)", + "value": ["66:41:0303161:100:265", "66:41:0303161:100:266"], + }, + ], + } + + +# ── Unit tests: schemas ─────────────────────────────────────────────────────── + + +def test_nspd_bulk_feature_parse_basic() -> None: + """NSPDBulkFeature парсит id, geometry, properties.""" + raw = { + "id": 42, + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [6700000.0, 7700000.0]}, + "properties": { + "categoryId": 36368, + "options": {"cad_num": "66:41:0303161:1", "area": 300.0}, + }, + } + feat = NSPDBulkFeature.model_validate(raw) + assert feat.id == 42 + assert feat.category_id == 36368 + assert feat.cad_num == "66:41:0303161:1" + assert feat.options.area == 300.0 + + +def test_nspd_bulk_feature_extra_fields_preserved() -> None: + """extra='allow' → неизвестные поля сохраняются.""" + raw = { + "id": 1, + "geometry": None, + "properties": {}, + "unknown_future_field": "value", + } + feat = NSPDBulkFeature.model_validate(raw) + assert feat.model_extra["unknown_future_field"] == "value" # type: ignore[index] + + +def test_nspd_options_effective_cad_num_prefers_cad_num() -> None: + """effective_cad_num берёт cad_num если оба заданы.""" + from app.schemas.nspd_bulk import NSPDOptions + + opts = NSPDOptions(cad_num="66:41:0303161:1", cad_number="66:41:0303161:2") + assert opts.effective_cad_num == "66:41:0303161:1" + + +def test_nspd_options_fallback_to_cad_number() -> None: + """effective_cad_num fallback → cad_number если cad_num is None.""" + from app.schemas.nspd_bulk import NSPDOptions + + opts = NSPDOptions(cad_number="66:41:0303161:2") + assert opts.effective_cad_num == "66:41:0303161:2" + + +def test_quarter_snapshot_overflow_categories() -> None: + """overflow_categories = категории с totalCount > 20.""" + snap = QuarterSnapshot( + quarter="66:41:0303161", + fetched_at="2026-05-15T00:00:00+00:00", + meta_counts={36368: 170, 36369: 256, 36383: 5, 39663: 1}, + ) + overflow = snap.overflow_categories + assert 36368 in overflow + assert 36369 in overflow + assert 36383 not in overflow # 5 <= 20 + assert 39663 not in overflow # 1 <= 20 + + +def test_objects_listing_defaults() -> None: + """ObjectsListing дефолтные значения.""" + listing = ObjectsListing(objdoc_id=12345) + assert listing.flats_count == 0 + assert listing.parking_count == 0 + assert listing.flats_cad_nums == [] + + +# ── Unit tests: client parse logic (mocked HTTP) ───────────────────────────── + + +@pytest.mark.asyncio +async def test_search_by_quarter_parses_features( + sample_quarter_response: dict[str, Any], +) -> None: + """search_by_quarter корректно парсит features и meta_counts.""" + async with NSPDBulkClient() as client: + with patch.object(client, "_get_json", new=AsyncMock(return_value=sample_quarter_response)): + snap = await client.search_by_quarter("66:41:0303161") + + assert snap.quarter == "66:41:0303161" + assert len(snap.features) == 2 + assert snap.meta_counts[36368] == 170 + assert snap.meta_counts[36369] == 256 + assert snap.meta_counts[37163] == 5877 + # overflow: категории с >20 + overflow = snap.overflow_categories + assert 36368 in overflow + assert 36369 in overflow + assert 36383 in overflow + assert 37163 in overflow + assert 39663 not in overflow # totalCount=1 → no overflow + + +@pytest.mark.asyncio +async def test_search_by_quarter_empty_on_404() -> None: + """search_by_quarter возвращает пустой snapshot при 404.""" + async with NSPDBulkClient() as client: + with patch.object( + client, + "_get_json", + new=AsyncMock(side_effect=NspdBulkError("HTTP 404: not found")), + ): + snap = await client.search_by_quarter("66:41:9999999") + + assert snap.features == [] + assert snap.meta_counts == {} + assert snap.quarter == "66:41:9999999" + + +@pytest.mark.asyncio +async def test_search_by_quarter_raises_on_waf() -> None: + """search_by_quarter пробрасывает NspdBulkWafError (403) без retry.""" + async with NSPDBulkClient() as client: + with patch.object( + client, + "_get_json", + new=AsyncMock(side_effect=NspdBulkWafError("HTTP 403 WAF")), + ): + with pytest.raises(NspdBulkWafError): + await client.search_by_quarter("66:41:0303161") + + +@pytest.mark.asyncio +async def test_wms_feature_info_parses_features( + sample_wms_response: dict[str, Any], +) -> None: + """wms_feature_info парсит features из GeoJSON response.""" + async with NSPDBulkClient() as client: + with patch.object(client, "_get_json", new=AsyncMock(return_value=sample_wms_response)): + features = await client.wms_feature_info( + layer_id=36368, + bbox=(6700000.0, 7700000.0, 6701000.0, 7701000.0), + click_xy=(256, 256), + ) + + assert len(features) == 1 + assert features[0].category_id == 36368 + assert features[0].cad_num == "66:41:0303161:789" + + +@pytest.mark.asyncio +async def test_wms_feature_info_empty_response() -> None: + """wms_feature_info возвращает [] при пустом response.""" + async with NSPDBulkClient() as client: + with patch.object(client, "_get_json", new=AsyncMock(return_value={"features": []})): + features = await client.wms_feature_info( + layer_id=36368, + bbox=(6700000.0, 7700000.0, 6701000.0, 7701000.0), + click_xy=(256, 256), + ) + assert features == [] + + +@pytest.mark.asyncio +async def test_list_objects_in_building_parses( + sample_tab_group_response: dict[str, Any], +) -> None: + """list_objects_in_building парсит counts и cad_nums.""" + async with NSPDBulkClient() as client: + with patch.object( + client, "_get_json", new=AsyncMock(return_value=sample_tab_group_response) + ): + listing = await client.list_objects_in_building(objdoc_id=42065602) + + assert listing.objdoc_id == 42065602 + assert listing.flats_count == 264 + assert listing.parking_count == 4 + assert "66:41:0303161:100:1" in listing.flats_cad_nums + assert "66:41:0303161:100:265" in listing.parking_cad_nums + + +@pytest.mark.asyncio +async def test_rate_limit_semaphore_max_3_concurrent() -> None: + """Не более 3 одновременных запросов через _SEMAPHORE. + + Мокируем httpx.AsyncClient.get (нижний уровень) с задержкой, чтобы + реальный семафор работал. Считаем max in-flight внутри семафора. + """ + import httpx + + import app.scrapers.nspd_bulk_client as bulk_mod + + max_concurrent = 0 + current = 0 + lock = asyncio.Lock() + call_count = 0 + + async def slow_get(*args: Any, **kwargs: Any) -> httpx.Response: + nonlocal max_concurrent, current, call_count + # Фиксируем вход — уже внутри семафора (httpx.get вызывается после async with _SEMAPHORE) + async with lock: + current += 1 + call_count += 1 + if current > max_concurrent: + max_concurrent = current + await asyncio.sleep(0.02) # симулируем I/O задержку + async with lock: + current -= 1 + return httpx.Response( + 200, + json={"data": {"features": [], "meta": []}}, + request=MagicMock(), + ) + + async with NSPDBulkClient() as client: + assert client._client is not None + with patch.object(client._client, "get", new=AsyncMock(side_effect=slow_get)): + tasks = [client.search_by_quarter(f"66:41:{i:07d}") for i in range(6)] + await asyncio.gather(*tasks) + + # _SEMAPHORE(3) → не более 3 одновременно внутри slow_get + assert max_concurrent <= 3, f"Expected ≤3 concurrent, got {max_concurrent}" + assert call_count == 6 # все 6 вызовов прошли + # Убедимся что _SEMAPHORE в модуле имеет нужное значение capacity + assert bulk_mod._SEMAPHORE._value >= 0 # семафор сброшен после всех задач + + +@pytest.mark.asyncio +async def test_http_403_raises_waf_no_retry() -> None: + """403 → NspdBulkWafError немедленно, без retry.""" + import httpx + + call_count = 0 + + async def fake_get(*args: Any, **kwargs: Any) -> httpx.Response: + nonlocal call_count + call_count += 1 + return httpx.Response(403, text="Forbidden", request=MagicMock()) + + async with NSPDBulkClient() as client: + assert client._client is not None + with patch.object(client._client, "get", new=AsyncMock(side_effect=fake_get)): + with pytest.raises(NspdBulkWafError): + await client._get_json("https://nspd.gov.ru/test") + + assert call_count == 1 # ровно 1 попытка, нет retry + + +@pytest.mark.asyncio +async def test_http_429_backoff_exhausted() -> None: + """429 → exponential backoff 3 retries → NspdBulkRateLimitError.""" + import httpx + + from app.scrapers.nspd_bulk_client import NspdBulkRateLimitError + + call_count = 0 + + async def fake_get(*args: Any, **kwargs: Any) -> httpx.Response: + nonlocal call_count + call_count += 1 + return httpx.Response(429, text="Too Many Requests", request=MagicMock()) + + # Патчим sleep чтобы тест не ждал 2+4+8 секунд + async with NSPDBulkClient(max_retries=3) as client: + assert client._client is not None + with ( + patch.object(client._client, "get", new=AsyncMock(side_effect=fake_get)), + patch("app.scrapers.nspd_bulk_client.asyncio.sleep", new=AsyncMock()), + ): + with pytest.raises(NspdBulkRateLimitError): + await client._get_json("https://nspd.gov.ru/test") + + # 1 оригинальный + 3 retry = 4 вызова + assert call_count == 4 + + +# ── E2E tests (network, skipped by default) ─────────────────────────────────── + +pytestmark_slow = pytest.mark.slow + + +@pytest.mark.slow +@pytest.mark.skip(reason="E2E: требует сеть NSPD — запускать вручную с -m slow") +@pytest.mark.asyncio +async def test_search_by_quarter_real_ekb() -> None: + """E2E: поиск по реальному кварталу ЕКБ — 66:41:0303161. + + Проверяет: + - features > 100 (квартал непустой) + - meta содержит ключи 36368/36369/36383 с totalCount > 0 + - overflow_categories не пуст (значит totalCount > 20 для хотя бы одной) + - headers корректны (нет 403/429) + """ + async with NSPDBulkClient() as client: + snap = await client.search_by_quarter("66:41:0303161") + + assert len(snap.features) > 0, "Ожидали хотя бы 1 feature" + # Meta должна содержать основные категории + assert 36368 in snap.meta_counts, "Нет categoryId 36368 (ЗУ ЕГРН) в meta" + assert 36369 in snap.meta_counts, "Нет categoryId 36369 (Здания) в meta" + assert snap.meta_counts[36368] > 0, "ЗУ ЕГРН totalCount должен быть > 0" + assert snap.meta_counts[36369] > 0, "Здания totalCount должен быть > 0" + # В этом квартале >170 ЗУ и >256 зданий → overflow + assert len(snap.overflow_categories) > 0, "Ожидали overflow_categories непустой" + # total_meta_count > 100 (квартал богатый данными) + assert snap.total_meta_count > 100 + + +@pytest.mark.slow +@pytest.mark.skip(reason="E2E: требует сеть NSPD — запускать вручную с -m slow") +@pytest.mark.asyncio +async def test_search_by_quarter_empty() -> None: + """E2E: несуществующий квартал → пустой snapshot.""" + async with NSPDBulkClient() as client: + snap = await client.search_by_quarter("66:41:9999999") + + # Должно вернуть пустой snapshot (не raise) + assert snap.features == [] + assert snap.quarter == "66:41:9999999" + + +@pytest.mark.slow +@pytest.mark.skip(reason="E2E: требует сеть NSPD — запускать вручную с -m slow") +@pytest.mark.asyncio +async def test_wms_feature_info_real() -> None: + """E2E: WMS GetFeatureInfo реальный bbox из ЕКБ. + + Использует bbox центра квартала 66:41:0303161 в EPSG:3857. + """ + # Центральная точка квартала 66:41:0303161 в 3857 (примерно) + # ЕКБ центр ≈ 60.6°E, 56.84°N → 3857 ≈ (6748000, 7772000) + center_x = 6748000.0 + center_y = 7772000.0 + half = 500.0 # 500м bbox + bbox = (center_x - half, center_y - half, center_x + half, center_y + half) + + async with NSPDBulkClient() as client: + features = await client.wms_feature_info( + layer_id=36368, # ЗУ ЕГРН + bbox=bbox, + click_xy=(256, 256), + width=512, + height=512, + ) + + # Хотя бы 1 feature в bbox центра ЕКБ квартала + assert len(features) >= 1, f"Ожидали ≥1 feature, получили {len(features)}" + + +@pytest.mark.slow +@pytest.mark.skip(reason="E2E: требует сеть NSPD — запускать вручную с -m slow") +@pytest.mark.asyncio +async def test_list_objects_in_building_real() -> None: + """E2E: реальный objdocId=42065602 → >150 flats_cad_nums.""" + async with NSPDBulkClient() as client: + listing = await client.list_objects_in_building(objdoc_id=42065602) + + assert listing.objdoc_id == 42065602 + assert listing.flats_count > 150, f"Ожидали >150 помещений, получили {listing.flats_count}" + assert ( + len(listing.flats_cad_nums) > 150 + ), f"Ожидали >150 cad_nums помещений, получили {len(listing.flats_cad_nums)}"