Add async NSPD client (nspd_bulk_client.py) with 3 new methods: - search_by_quarter: REST /api/geoportal/v2/search/geoportal — snapshot 20 per cat + meta totals - wms_feature_info: WMS GetFeatureInfo для grid-walk discovery - list_objects_in_building: tab-group-data (Q3 deferred — код ready) Rate limit: asyncio.Semaphore(3) + 0.05s jitter. 429 → exponential backoff 2→4→8s, max 3 retries. 403 (WAF) → raise immediately, no retry. Pydantic schemas (nspd_bulk.py): NSPDBulkFeature, QuarterSnapshot, ObjectsListing with extra='allow' to preserve unknown fields → raw_props JSONB downstream. 15 unit tests + 4 e2e (real NSPD, skipped in CI by default). Co-authored-by: lekss361 <claudestars@proton.me>
This commit is contained in:
lekss361
GitHub
parent
b4327635fa
commit
b7889d113b
4 changed files with 1032 additions and 0 deletions
134
backend/app/schemas/nspd_bulk.py
Normal file
134
backend/app/schemas/nspd_bulk.py
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
"""Pydantic schemas для NSPD bulk cadastre ingest (PR 2/5, issue #168).
|
||||
|
||||
Отдельный модуль от legacy NSPDFeature dataclass (nspd_client.py) —
|
||||
там frozen dataclass для on-demand flow, тут Pydantic для bulk pipeline
|
||||
с extra='allow' чтобы не потерять неизвестные поля → raw_props JSONB.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
|
||||
class NSPDOptions(BaseModel):
|
||||
"""Опции кадастрового объекта из NSPD.
|
||||
|
||||
Все поля Optional — разные categoryId возвращают разные subsets.
|
||||
extra='allow' → любые незнакомые поля сохраняются (forward-compatible).
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||||
|
||||
cad_num: str | None = None
|
||||
cad_number: str | None = None # некоторые категории используют cad_number
|
||||
quarter_cad_number: str | None = None
|
||||
parent_cad_number: str | None = None
|
||||
area: float | None = None
|
||||
cost_value: float | None = None
|
||||
cost_index: float | None = None
|
||||
purpose: str | None = None
|
||||
floors: str | int | None = None
|
||||
year_built: str | int | None = None
|
||||
readable_address: str | None = None
|
||||
ownership_type: str | None = None
|
||||
right_type: str | None = None
|
||||
status: str | None = None
|
||||
land_record_area: float | None = None
|
||||
specified_area: float | None = None
|
||||
declared_area: float | None = None
|
||||
permitted_use_established_by_document: str | None = None
|
||||
category_type: str | None = None
|
||||
build_record_type: str | None = None
|
||||
build_record_type_value: str | None = None
|
||||
objdoc_id: int | None = None
|
||||
registers_id: int | None = None
|
||||
okato: str | None = None
|
||||
kladr: str | None = None
|
||||
|
||||
@property
|
||||
def effective_cad_num(self) -> str | None:
|
||||
"""Нормализованный кад. номер: cad_num или cad_number (разные категории)."""
|
||||
return self.cad_num or self.cad_number
|
||||
|
||||
|
||||
class NSPDBulkFeature(BaseModel):
|
||||
"""GeoJSON Feature из NSPD bulk endpoints.
|
||||
|
||||
Pydantic-версия (не dataclass) — для удобной сериализации + extra='allow'.
|
||||
geometry — GeoJSON-dict в EPSG:3857 (transform на стороне Celery-таска).
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||||
|
||||
id: int | str | None = None
|
||||
type: str = "Feature"
|
||||
geometry: dict[str, Any] | None = None
|
||||
properties: dict[str, Any] = {}
|
||||
|
||||
@property
|
||||
def category_id(self) -> int | None:
|
||||
"""categoryId из properties (ключ для routing к target table)."""
|
||||
val = self.properties.get("categoryId")
|
||||
return int(val) if val is not None else None
|
||||
|
||||
@property
|
||||
def options(self) -> NSPDOptions:
|
||||
"""Распарсить options из properties.options если есть, иначе пустой."""
|
||||
opts = self.properties.get("options") or {}
|
||||
return NSPDOptions.model_validate(opts)
|
||||
|
||||
@property
|
||||
def cad_num(self) -> str | None:
|
||||
"""Shortcut: cad_num из options (часто нужен в caller)."""
|
||||
return self.options.effective_cad_num
|
||||
|
||||
|
||||
class QuarterSnapshot(BaseModel):
|
||||
"""Результат search_by_quarter — snapshot данных для одного квартала.
|
||||
|
||||
Первые 20 объектов на категорию из NSPD search endpoint.
|
||||
overflow_categories — категории требующие grid-walk (totalCount > 20).
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
quarter: str
|
||||
fetched_at: str # ISO datetime string
|
||||
features: list[NSPDBulkFeature] = []
|
||||
meta_counts: dict[int, int] = {} # {categoryId: totalCount}
|
||||
|
||||
@property
|
||||
def overflow_categories(self) -> list[int]:
|
||||
"""Категории где totalCount > 20 — нужен grid-walk для полноты."""
|
||||
return [cat_id for cat_id, count in self.meta_counts.items() if count > 20]
|
||||
|
||||
@property
|
||||
def total_meta_count(self) -> int:
|
||||
"""Суммарное количество объектов по meta (до cap)."""
|
||||
return sum(self.meta_counts.values())
|
||||
|
||||
|
||||
class ObjectsListing(BaseModel):
|
||||
"""Результат list_objects_in_building — список помещений и машино-мест здания.
|
||||
|
||||
Получается через /api/geoportal/v1/tab-group-data?tabClass=objectsList.
|
||||
Q3 deferred — код готов, но в bulk_harvest_quarter MVP не используется.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
objdoc_id: int
|
||||
flats_count: int = 0
|
||||
parking_count: int = 0
|
||||
flats_cad_nums: list[str] = []
|
||||
parking_cad_nums: list[str] = []
|
||||
|
||||
|
||||
__all__ = [
|
||||
"NSPDBulkFeature",
|
||||
"NSPDOptions",
|
||||
"ObjectsListing",
|
||||
"QuarterSnapshot",
|
||||
]
|
||||
407
backend/app/scrapers/nspd_bulk_client.py
Normal file
407
backend/app/scrapers/nspd_bulk_client.py
Normal file
|
|
@ -0,0 +1,407 @@
|
|||
"""NSPD bulk cadastre client — async httpx-based, rate-limited.
|
||||
|
||||
PR 2/5 of GH issue #168: Bulk Cadastre Ingest (ЕКБ 2408 кварталов).
|
||||
|
||||
Три новых метода:
|
||||
- search_by_quarter(quarter) → QuarterSnapshot (phase 1 snapshot)
|
||||
- wms_feature_info(layer_id, bbox, click_xy) → list[NSPDBulkFeature] (grid-walk)
|
||||
- list_objects_in_building(objdoc_id) → ObjectsListing (Q3 deferred)
|
||||
|
||||
Ключевые отличия от nspd_client.py (sync urllib):
|
||||
- async httpx.AsyncClient (connection pooling, timeout objects)
|
||||
- asyncio.Semaphore(3) — не более 3 одновременных запросов
|
||||
- asyncio.sleep(0.05) jitter между запросами
|
||||
- 429 → exponential backoff (2 → 4 → 8 с, max 3 retries)
|
||||
- 403 WAF → raise немедленно (не retry, нужен manual investigate)
|
||||
|
||||
Geometry: NSPD возвращает geom в EPSG:3857. ST_Transform(geom, 4326)
|
||||
делается на стороне Celery-таска (PR 3) при upsert в DB.
|
||||
|
||||
Rate limit: 3 req/s sustained через Semaphore(3) + 0.05s jitter.
|
||||
При 429 — exponential backoff: 2s → 4s → 8s, max 3 retries.
|
||||
При 403 (WAF) — raise NspdBulkWafError, не retry.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import datetime as _dt
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.schemas.nspd_bulk import NSPDBulkFeature, ObjectsListing, QuarterSnapshot
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── Константы ──────────────────────────────────────────────────────────────────
|
||||
|
||||
NSPD_BASE = "https://nspd.gov.ru/api"
|
||||
NSPD_SEARCH_URL = f"{NSPD_BASE}/geoportal/v2/search/geoportal"
|
||||
NSPD_WMS_BASE = f"{NSPD_BASE}/aeggis/v4"
|
||||
NSPD_TAB_GROUP_URL = f"{NSPD_BASE}/geoportal/v1/tab-group-data"
|
||||
|
||||
# Лимит одновременных запросов (глобальный, shared между всеми instance'ами)
|
||||
_SEMAPHORE = asyncio.Semaphore(3)
|
||||
|
||||
# Таймаут подключения и чтения
|
||||
DEFAULT_TIMEOUT = httpx.Timeout(30.0, connect=10.0)
|
||||
|
||||
# Headers из HAR audit 2026-05-15
|
||||
DEFAULT_HEADERS: dict[str, str] = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Referer": "https://nspd.gov.ru/map",
|
||||
"Accept": "*/*",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
"x-timezone": "Europe/Moscow",
|
||||
# Origin только для POST endpoints — не включаем для GET (лишний header)
|
||||
}
|
||||
|
||||
# Retry settings
|
||||
_MAX_RETRIES = 3
|
||||
_BACKOFF_BASE_S = 2.0 # 2 → 4 → 8 секунд
|
||||
|
||||
|
||||
# ── Exceptions ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class NspdBulkError(RuntimeError):
|
||||
"""Неуспешный ответ от NSPD bulk endpoint."""
|
||||
|
||||
|
||||
class NspdBulkWafError(NspdBulkError):
|
||||
"""HTTP 403 WAF блокировка — не retry, нужно manual investigate."""
|
||||
|
||||
|
||||
class NspdBulkRateLimitError(NspdBulkError):
|
||||
"""HTTP 429 Rate limit — исчерпаны retries после backoff."""
|
||||
|
||||
|
||||
# ── NSPDBulkClient ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class NSPDBulkClient:
|
||||
"""Async NSPD client для bulk cadastre ingest.
|
||||
|
||||
Использовать как async context manager:
|
||||
|
||||
async with NSPDBulkClient() as client:
|
||||
snapshot = await client.search_by_quarter("66:41:0303161")
|
||||
|
||||
Или с explicit lifecycle (для Celery tasks):
|
||||
|
||||
client = NSPDBulkClient()
|
||||
await client.__aenter__()
|
||||
try:
|
||||
...
|
||||
finally:
|
||||
await client.__aexit__(None, None, None)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
timeout: httpx.Timeout = DEFAULT_TIMEOUT,
|
||||
headers: dict[str, str] | None = None,
|
||||
max_retries: int = _MAX_RETRIES,
|
||||
) -> None:
|
||||
self._timeout = timeout
|
||||
self._headers = headers or DEFAULT_HEADERS
|
||||
self._max_retries = max_retries
|
||||
self._client: httpx.AsyncClient | None = None
|
||||
|
||||
async def __aenter__(self) -> NSPDBulkClient:
|
||||
self._client = httpx.AsyncClient(
|
||||
headers=self._headers,
|
||||
timeout=self._timeout,
|
||||
follow_redirects=True,
|
||||
)
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *_: Any) -> None:
|
||||
if self._client is not None:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
# ── Internal HTTP helper ───────────────────────────────────────────────────
|
||||
|
||||
async def _get_json(self, url: str, params: dict[str, Any] | None = None) -> Any:
|
||||
"""GET → JSON с rate-limit semaphore, jitter, retry при 429.
|
||||
|
||||
Raises:
|
||||
NspdBulkWafError при 403.
|
||||
NspdBulkRateLimitError при 429 после исчерпания retries.
|
||||
NspdBulkError при прочих 4xx/5xx.
|
||||
"""
|
||||
if self._client is None:
|
||||
raise RuntimeError("NSPDBulkClient не инициализирован — используй async with")
|
||||
|
||||
attempt = 0
|
||||
while True:
|
||||
async with _SEMAPHORE:
|
||||
# Небольшой jitter чтобы N одновременных запросов не начинались в 0мс
|
||||
await asyncio.sleep(0.05)
|
||||
try:
|
||||
resp = await self._client.get(url, params=params)
|
||||
except httpx.TransportError as e:
|
||||
raise NspdBulkError(f"Network error: {e}") from e
|
||||
|
||||
# Обработка статусов — вне семафора чтобы не держать слот при sleep
|
||||
if resp.status_code == 403:
|
||||
body_preview = resp.text[:200]
|
||||
logger.warning("NSPD WAF 403 blocked: url=%s preview=%s", url, body_preview)
|
||||
raise NspdBulkWafError(f"HTTP 403 WAF block (manual investigate required): {url}")
|
||||
|
||||
if resp.status_code == 429:
|
||||
attempt += 1
|
||||
if attempt > self._max_retries:
|
||||
raise NspdBulkRateLimitError(
|
||||
f"HTTP 429 rate-limit: exhausted {self._max_retries} retries for {url}"
|
||||
)
|
||||
backoff = _BACKOFF_BASE_S * (2 ** (attempt - 1))
|
||||
logger.warning(
|
||||
"NSPD 429 rate-limit: retry %d/%d after %.0fs backoff, url=%s",
|
||||
attempt,
|
||||
self._max_retries,
|
||||
backoff,
|
||||
url,
|
||||
)
|
||||
await asyncio.sleep(backoff)
|
||||
continue
|
||||
|
||||
if resp.status_code >= 400:
|
||||
body_preview = resp.text[:300]
|
||||
raise NspdBulkError(f"HTTP {resp.status_code}: {url} — {body_preview}")
|
||||
|
||||
return resp.json()
|
||||
|
||||
# ── 1. search_by_quarter ──────────────────────────────────────────────────
|
||||
|
||||
async def search_by_quarter(self, quarter: str) -> QuarterSnapshot:
|
||||
"""Поиск объектов в квартале через NSPD search endpoint.
|
||||
|
||||
GET /api/geoportal/v2/search/geoportal?thematicSearchId=1&query={quarter}
|
||||
|
||||
Возвращает первые 20 объектов на категорию + meta с totalCount.
|
||||
Категории с totalCount > 20 попадают в overflow_categories
|
||||
и требуют grid-walk (wms_feature_info) для полноты.
|
||||
|
||||
Args:
|
||||
quarter: кадастровый номер квартала в формате `66:41:NNNNNN`.
|
||||
|
||||
Returns:
|
||||
QuarterSnapshot с features + meta_counts.
|
||||
При 404 или пустом ответе — snapshot с features=[].
|
||||
|
||||
Raises:
|
||||
NspdBulkWafError при 403.
|
||||
NspdBulkError при сетевых / прочих ошибках.
|
||||
"""
|
||||
fetched_at = _dt.datetime.now(_dt.UTC).isoformat()
|
||||
|
||||
try:
|
||||
data = await self._get_json(
|
||||
NSPD_SEARCH_URL,
|
||||
params={"thematicSearchId": 1, "query": quarter},
|
||||
)
|
||||
except NspdBulkError as e:
|
||||
# 404 или пустой ответ → возвращаем пустой snapshot
|
||||
err_str = str(e)
|
||||
if "HTTP 404" in err_str or "HTTP 400" in err_str:
|
||||
logger.info(
|
||||
"search_by_quarter: quarter %s not found (404/400), returning empty",
|
||||
quarter,
|
||||
)
|
||||
return QuarterSnapshot(quarter=quarter, fetched_at=fetched_at)
|
||||
raise
|
||||
|
||||
# Парсим response
|
||||
# Shape: {"data": {"type": "FeatureCollection", "features": [...], "meta": [...]}, ...}
|
||||
# или: {"data": {"features": [...], "meta": [...]}}
|
||||
payload = data if isinstance(data, dict) else {}
|
||||
data_block = payload.get("data") or {}
|
||||
if isinstance(data_block, str):
|
||||
# Иногда NSPD оборачивает data в строку (Bug_Nspd_Geo_Str_Object_No_Get_Fixed)
|
||||
logger.warning(
|
||||
"search_by_quarter: data block is string for quarter %s, returning empty",
|
||||
quarter,
|
||||
)
|
||||
return QuarterSnapshot(quarter=quarter, fetched_at=fetched_at)
|
||||
|
||||
raw_features: list[dict[str, Any]] = data_block.get("features") or []
|
||||
raw_meta: list[dict[str, Any]] = data_block.get("meta") or []
|
||||
|
||||
features = [NSPDBulkFeature.model_validate(f) for f in raw_features]
|
||||
|
||||
# meta: [{categoryId: N, totalCount: M}, ...]
|
||||
meta_counts: dict[int, int] = {}
|
||||
for m in raw_meta:
|
||||
cat_id = m.get("categoryId")
|
||||
total = m.get("totalCount")
|
||||
if cat_id is not None and total is not None:
|
||||
meta_counts[int(cat_id)] = int(total)
|
||||
|
||||
logger.info(
|
||||
"search_by_quarter: quarter=%s features=%d meta_cats=%d overflow=%d",
|
||||
quarter,
|
||||
len(features),
|
||||
len(meta_counts),
|
||||
len([c for c, cnt in meta_counts.items() if cnt > 20]),
|
||||
)
|
||||
|
||||
return QuarterSnapshot(
|
||||
quarter=quarter,
|
||||
fetched_at=fetched_at,
|
||||
features=features,
|
||||
meta_counts=meta_counts,
|
||||
)
|
||||
|
||||
# ── 2. wms_feature_info ───────────────────────────────────────────────────
|
||||
|
||||
async def wms_feature_info(
|
||||
self,
|
||||
layer_id: int,
|
||||
bbox: tuple[float, float, float, float],
|
||||
click_xy: tuple[int, int],
|
||||
width: int = 512,
|
||||
height: int = 512,
|
||||
) -> list[NSPDBulkFeature]:
|
||||
"""WMS GetFeatureInfo — объекты layer'а в bbox на pixel-клике.
|
||||
|
||||
GET /api/aeggis/v4/{layer_id}/wms?REQUEST=GetFeatureInfo&...
|
||||
|
||||
Используется для grid-walk фазы: каждая ячейка bbox квартала
|
||||
кликается в центре (click_xy = (width//2, height//2)).
|
||||
|
||||
Args:
|
||||
layer_id: NSPD layer ID (например 36368 для ЗУ ЕГРН).
|
||||
bbox: (x_min, y_min, x_max, y_max) в EPSG:3857 (метры).
|
||||
click_xy: (I, J) пиксель-координаты клика в [0..width-1, 0..height-1].
|
||||
width: ширина виртуального tile.
|
||||
height: высота виртуального tile.
|
||||
|
||||
Returns:
|
||||
Список NSPDBulkFeature (может быть пуст).
|
||||
|
||||
Raises:
|
||||
NspdBulkWafError при 403.
|
||||
NspdBulkError при прочих ошибках.
|
||||
"""
|
||||
xmin, ymin, xmax, ymax = bbox
|
||||
click_i, click_j = click_xy
|
||||
|
||||
url = f"{NSPD_WMS_BASE}/{layer_id}/wms"
|
||||
params: dict[str, Any] = {
|
||||
"REQUEST": "GetFeatureInfo",
|
||||
"SERVICE": "WMS",
|
||||
"VERSION": "1.3.0",
|
||||
"INFO_FORMAT": "application/json",
|
||||
"LAYERS": str(layer_id),
|
||||
"QUERY_LAYERS": str(layer_id),
|
||||
"CRS": "EPSG:3857",
|
||||
"BBOX": f"{xmin},{ymin},{xmax},{ymax}",
|
||||
"WIDTH": str(width),
|
||||
"HEIGHT": str(height),
|
||||
"I": str(click_i),
|
||||
"J": str(click_j),
|
||||
"STYLES": "",
|
||||
"TRANSPARENT": "true",
|
||||
"FORMAT": "image/png",
|
||||
"FEATURE_COUNT": "10",
|
||||
}
|
||||
|
||||
data = await self._get_json(url, params=params)
|
||||
raw_features: list[dict[str, Any]] = (data or {}).get("features") or []
|
||||
return [NSPDBulkFeature.model_validate(f) for f in raw_features]
|
||||
|
||||
# ── 3. list_objects_in_building ───────────────────────────────────────────
|
||||
# Q3 deferred — метод реализован, но не вызывается в bulk_harvest_quarter MVP.
|
||||
# Готов для per-building помещения/парковка фазы.
|
||||
|
||||
async def list_objects_in_building(
|
||||
self,
|
||||
objdoc_id: int,
|
||||
registers_id: int = 36441,
|
||||
) -> ObjectsListing:
|
||||
"""Список помещений и машино-мест в здании.
|
||||
|
||||
GET /api/geoportal/v1/tab-group-data?tabClass=objectsList
|
||||
&objdocId={objdoc_id}®istersId={registers_id}
|
||||
|
||||
Response shape:
|
||||
{title: "Список объектов", object: [
|
||||
{title: "Помещения (количество)", value: ["264"]},
|
||||
{title: "Машино-места (количество)", value: ["4"]},
|
||||
{title: "Помещения (список)", value: [...cad_nums]},
|
||||
{title: "Машино-места (список)", value: [...cad_nums]},
|
||||
]}
|
||||
|
||||
NOTE: Q3 deferred — метод готов, но НЕ вызывается в PR 2/3 MVP.
|
||||
Использовать при реализации bulk_harvest_flats (отдельный task).
|
||||
|
||||
Args:
|
||||
objdoc_id: NSPD internal object document ID (из NSPDOptions.objdoc_id).
|
||||
registers_id: NSPD registers ID (36441 — здания ЕГРН по умолчанию).
|
||||
|
||||
Returns:
|
||||
ObjectsListing с counts и cad_nums списками.
|
||||
|
||||
Raises:
|
||||
NspdBulkWafError при 403.
|
||||
NspdBulkError при прочих ошибках.
|
||||
"""
|
||||
data = await self._get_json(
|
||||
NSPD_TAB_GROUP_URL,
|
||||
params={
|
||||
"tabClass": "objectsList",
|
||||
"objdocId": str(objdoc_id),
|
||||
"registersId": str(registers_id),
|
||||
},
|
||||
)
|
||||
|
||||
flats_count = 0
|
||||
parking_count = 0
|
||||
flats_cad_nums: list[str] = []
|
||||
parking_cad_nums: list[str] = []
|
||||
|
||||
obj_list: list[dict[str, Any]] = (data or {}).get("object") or []
|
||||
for entry in obj_list:
|
||||
title: str = (entry.get("title") or "").lower()
|
||||
values: list[Any] = entry.get("value") or []
|
||||
|
||||
if "помещения" in title and "количество" in title:
|
||||
try:
|
||||
flats_count = int(values[0]) if values else 0
|
||||
except (ValueError, TypeError):
|
||||
flats_count = 0
|
||||
|
||||
elif "машино-места" in title and "количество" in title:
|
||||
try:
|
||||
parking_count = int(values[0]) if values else 0
|
||||
except (ValueError, TypeError):
|
||||
parking_count = 0
|
||||
|
||||
elif "помещения" in title and "список" in title:
|
||||
flats_cad_nums = [str(v) for v in values if v]
|
||||
|
||||
elif "машино-места" in title and "список" in title:
|
||||
parking_cad_nums = [str(v) for v in values if v]
|
||||
|
||||
return ObjectsListing(
|
||||
objdoc_id=objdoc_id,
|
||||
flats_count=flats_count,
|
||||
parking_count=parking_count,
|
||||
flats_cad_nums=flats_cad_nums,
|
||||
parking_cad_nums=parking_cad_nums,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"NSPDBulkClient",
|
||||
"NspdBulkError",
|
||||
"NspdBulkRateLimitError",
|
||||
"NspdBulkWafError",
|
||||
]
|
||||
0
backend/tests/scrapers/__init__.py
Normal file
0
backend/tests/scrapers/__init__.py
Normal file
491
backend/tests/scrapers/test_nspd_bulk_client.py
Normal file
491
backend/tests/scrapers/test_nspd_bulk_client.py
Normal file
|
|
@ -0,0 +1,491 @@
|
|||
"""Тесты для NSPDBulkClient (nspd_bulk_client.py) — PR 2/5 issue #168.
|
||||
|
||||
Структура:
|
||||
- Unit тесты (без сети) — параметры, семафор, parse-логика
|
||||
- E2E тесты (@pytest.mark.slow + @pytest.mark.skip по умолчанию) —
|
||||
реальные запросы к NSPD, CI не дёргает
|
||||
|
||||
Запуск e2e вручную:
|
||||
cd backend && uv run pytest tests/scrapers/test_nspd_bulk_client.py -m slow -s -v
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.schemas.nspd_bulk import NSPDBulkFeature, ObjectsListing, QuarterSnapshot
|
||||
from app.scrapers.nspd_bulk_client import (
|
||||
NSPDBulkClient,
|
||||
NspdBulkError,
|
||||
NspdBulkWafError,
|
||||
)
|
||||
|
||||
# ── Fixtures ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_quarter_response() -> dict[str, Any]:
|
||||
"""Минимальный валидный ответ search_by_quarter из NSPD."""
|
||||
return {
|
||||
"data": {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"id": 1001,
|
||||
"type": "Feature",
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[
|
||||
[6700000.0, 7700000.0],
|
||||
[6701000.0, 7700000.0],
|
||||
[6701000.0, 7701000.0],
|
||||
[6700000.0, 7701000.0],
|
||||
[6700000.0, 7700000.0],
|
||||
],
|
||||
],
|
||||
},
|
||||
"properties": {
|
||||
"categoryId": 36368,
|
||||
"categoryName": "Земельные участки ЕГРН",
|
||||
"options": {
|
||||
"cad_num": "66:41:0303161:123",
|
||||
"area": 500.0,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": 2002,
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [6700500.0, 7700500.0]},
|
||||
"properties": {
|
||||
"categoryId": 36369,
|
||||
"categoryName": "Здания",
|
||||
"options": {
|
||||
"cad_num": "66:41:0303161:456:1",
|
||||
"floors": "5",
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
"meta": [
|
||||
{"categoryId": 36368, "totalCount": 170},
|
||||
{"categoryId": 36369, "totalCount": 256},
|
||||
{"categoryId": 36383, "totalCount": 75},
|
||||
{"categoryId": 37163, "totalCount": 5877},
|
||||
{"categoryId": 39663, "totalCount": 1},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_wms_response() -> dict[str, Any]:
|
||||
"""Минимальный WMS GetFeatureInfo response."""
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"id": 3003,
|
||||
"type": "Feature",
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[
|
||||
[6700100.0, 7700100.0],
|
||||
[6700900.0, 7700100.0],
|
||||
[6700900.0, 7700900.0],
|
||||
[6700100.0, 7700900.0],
|
||||
[6700100.0, 7700100.0],
|
||||
],
|
||||
],
|
||||
},
|
||||
"properties": {
|
||||
"categoryId": 36368,
|
||||
"options": {"cad_num": "66:41:0303161:789"},
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_tab_group_response() -> dict[str, Any]:
|
||||
"""Минимальный tab-group-data response для здания."""
|
||||
return {
|
||||
"title": "Список объектов",
|
||||
"object": [
|
||||
{"title": "Помещения (количество)", "value": ["264"]},
|
||||
{"title": "Машино-места (количество)", "value": ["4"]},
|
||||
{
|
||||
"title": "Помещения (список)",
|
||||
"value": ["66:41:0303161:100:1", "66:41:0303161:100:2"],
|
||||
},
|
||||
{
|
||||
"title": "Машино-места (список)",
|
||||
"value": ["66:41:0303161:100:265", "66:41:0303161:100:266"],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
# ── Unit tests: schemas ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_nspd_bulk_feature_parse_basic() -> None:
|
||||
"""NSPDBulkFeature парсит id, geometry, properties."""
|
||||
raw = {
|
||||
"id": 42,
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [6700000.0, 7700000.0]},
|
||||
"properties": {
|
||||
"categoryId": 36368,
|
||||
"options": {"cad_num": "66:41:0303161:1", "area": 300.0},
|
||||
},
|
||||
}
|
||||
feat = NSPDBulkFeature.model_validate(raw)
|
||||
assert feat.id == 42
|
||||
assert feat.category_id == 36368
|
||||
assert feat.cad_num == "66:41:0303161:1"
|
||||
assert feat.options.area == 300.0
|
||||
|
||||
|
||||
def test_nspd_bulk_feature_extra_fields_preserved() -> None:
|
||||
"""extra='allow' → неизвестные поля сохраняются."""
|
||||
raw = {
|
||||
"id": 1,
|
||||
"geometry": None,
|
||||
"properties": {},
|
||||
"unknown_future_field": "value",
|
||||
}
|
||||
feat = NSPDBulkFeature.model_validate(raw)
|
||||
assert feat.model_extra["unknown_future_field"] == "value" # type: ignore[index]
|
||||
|
||||
|
||||
def test_nspd_options_effective_cad_num_prefers_cad_num() -> None:
|
||||
"""effective_cad_num берёт cad_num если оба заданы."""
|
||||
from app.schemas.nspd_bulk import NSPDOptions
|
||||
|
||||
opts = NSPDOptions(cad_num="66:41:0303161:1", cad_number="66:41:0303161:2")
|
||||
assert opts.effective_cad_num == "66:41:0303161:1"
|
||||
|
||||
|
||||
def test_nspd_options_fallback_to_cad_number() -> None:
|
||||
"""effective_cad_num fallback → cad_number если cad_num is None."""
|
||||
from app.schemas.nspd_bulk import NSPDOptions
|
||||
|
||||
opts = NSPDOptions(cad_number="66:41:0303161:2")
|
||||
assert opts.effective_cad_num == "66:41:0303161:2"
|
||||
|
||||
|
||||
def test_quarter_snapshot_overflow_categories() -> None:
|
||||
"""overflow_categories = категории с totalCount > 20."""
|
||||
snap = QuarterSnapshot(
|
||||
quarter="66:41:0303161",
|
||||
fetched_at="2026-05-15T00:00:00+00:00",
|
||||
meta_counts={36368: 170, 36369: 256, 36383: 5, 39663: 1},
|
||||
)
|
||||
overflow = snap.overflow_categories
|
||||
assert 36368 in overflow
|
||||
assert 36369 in overflow
|
||||
assert 36383 not in overflow # 5 <= 20
|
||||
assert 39663 not in overflow # 1 <= 20
|
||||
|
||||
|
||||
def test_objects_listing_defaults() -> None:
|
||||
"""ObjectsListing дефолтные значения."""
|
||||
listing = ObjectsListing(objdoc_id=12345)
|
||||
assert listing.flats_count == 0
|
||||
assert listing.parking_count == 0
|
||||
assert listing.flats_cad_nums == []
|
||||
|
||||
|
||||
# ── Unit tests: client parse logic (mocked HTTP) ─────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_by_quarter_parses_features(
|
||||
sample_quarter_response: dict[str, Any],
|
||||
) -> None:
|
||||
"""search_by_quarter корректно парсит features и meta_counts."""
|
||||
async with NSPDBulkClient() as client:
|
||||
with patch.object(client, "_get_json", new=AsyncMock(return_value=sample_quarter_response)):
|
||||
snap = await client.search_by_quarter("66:41:0303161")
|
||||
|
||||
assert snap.quarter == "66:41:0303161"
|
||||
assert len(snap.features) == 2
|
||||
assert snap.meta_counts[36368] == 170
|
||||
assert snap.meta_counts[36369] == 256
|
||||
assert snap.meta_counts[37163] == 5877
|
||||
# overflow: категории с >20
|
||||
overflow = snap.overflow_categories
|
||||
assert 36368 in overflow
|
||||
assert 36369 in overflow
|
||||
assert 36383 in overflow
|
||||
assert 37163 in overflow
|
||||
assert 39663 not in overflow # totalCount=1 → no overflow
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_by_quarter_empty_on_404() -> None:
|
||||
"""search_by_quarter возвращает пустой snapshot при 404."""
|
||||
async with NSPDBulkClient() as client:
|
||||
with patch.object(
|
||||
client,
|
||||
"_get_json",
|
||||
new=AsyncMock(side_effect=NspdBulkError("HTTP 404: not found")),
|
||||
):
|
||||
snap = await client.search_by_quarter("66:41:9999999")
|
||||
|
||||
assert snap.features == []
|
||||
assert snap.meta_counts == {}
|
||||
assert snap.quarter == "66:41:9999999"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_by_quarter_raises_on_waf() -> None:
|
||||
"""search_by_quarter пробрасывает NspdBulkWafError (403) без retry."""
|
||||
async with NSPDBulkClient() as client:
|
||||
with patch.object(
|
||||
client,
|
||||
"_get_json",
|
||||
new=AsyncMock(side_effect=NspdBulkWafError("HTTP 403 WAF")),
|
||||
):
|
||||
with pytest.raises(NspdBulkWafError):
|
||||
await client.search_by_quarter("66:41:0303161")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wms_feature_info_parses_features(
|
||||
sample_wms_response: dict[str, Any],
|
||||
) -> None:
|
||||
"""wms_feature_info парсит features из GeoJSON response."""
|
||||
async with NSPDBulkClient() as client:
|
||||
with patch.object(client, "_get_json", new=AsyncMock(return_value=sample_wms_response)):
|
||||
features = await client.wms_feature_info(
|
||||
layer_id=36368,
|
||||
bbox=(6700000.0, 7700000.0, 6701000.0, 7701000.0),
|
||||
click_xy=(256, 256),
|
||||
)
|
||||
|
||||
assert len(features) == 1
|
||||
assert features[0].category_id == 36368
|
||||
assert features[0].cad_num == "66:41:0303161:789"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wms_feature_info_empty_response() -> None:
|
||||
"""wms_feature_info возвращает [] при пустом response."""
|
||||
async with NSPDBulkClient() as client:
|
||||
with patch.object(client, "_get_json", new=AsyncMock(return_value={"features": []})):
|
||||
features = await client.wms_feature_info(
|
||||
layer_id=36368,
|
||||
bbox=(6700000.0, 7700000.0, 6701000.0, 7701000.0),
|
||||
click_xy=(256, 256),
|
||||
)
|
||||
assert features == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_objects_in_building_parses(
|
||||
sample_tab_group_response: dict[str, Any],
|
||||
) -> None:
|
||||
"""list_objects_in_building парсит counts и cad_nums."""
|
||||
async with NSPDBulkClient() as client:
|
||||
with patch.object(
|
||||
client, "_get_json", new=AsyncMock(return_value=sample_tab_group_response)
|
||||
):
|
||||
listing = await client.list_objects_in_building(objdoc_id=42065602)
|
||||
|
||||
assert listing.objdoc_id == 42065602
|
||||
assert listing.flats_count == 264
|
||||
assert listing.parking_count == 4
|
||||
assert "66:41:0303161:100:1" in listing.flats_cad_nums
|
||||
assert "66:41:0303161:100:265" in listing.parking_cad_nums
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rate_limit_semaphore_max_3_concurrent() -> None:
|
||||
"""Не более 3 одновременных запросов через _SEMAPHORE.
|
||||
|
||||
Мокируем httpx.AsyncClient.get (нижний уровень) с задержкой, чтобы
|
||||
реальный семафор работал. Считаем max in-flight внутри семафора.
|
||||
"""
|
||||
import httpx
|
||||
|
||||
import app.scrapers.nspd_bulk_client as bulk_mod
|
||||
|
||||
max_concurrent = 0
|
||||
current = 0
|
||||
lock = asyncio.Lock()
|
||||
call_count = 0
|
||||
|
||||
async def slow_get(*args: Any, **kwargs: Any) -> httpx.Response:
|
||||
nonlocal max_concurrent, current, call_count
|
||||
# Фиксируем вход — уже внутри семафора (httpx.get вызывается после async with _SEMAPHORE)
|
||||
async with lock:
|
||||
current += 1
|
||||
call_count += 1
|
||||
if current > max_concurrent:
|
||||
max_concurrent = current
|
||||
await asyncio.sleep(0.02) # симулируем I/O задержку
|
||||
async with lock:
|
||||
current -= 1
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"data": {"features": [], "meta": []}},
|
||||
request=MagicMock(),
|
||||
)
|
||||
|
||||
async with NSPDBulkClient() as client:
|
||||
assert client._client is not None
|
||||
with patch.object(client._client, "get", new=AsyncMock(side_effect=slow_get)):
|
||||
tasks = [client.search_by_quarter(f"66:41:{i:07d}") for i in range(6)]
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
# _SEMAPHORE(3) → не более 3 одновременно внутри slow_get
|
||||
assert max_concurrent <= 3, f"Expected ≤3 concurrent, got {max_concurrent}"
|
||||
assert call_count == 6 # все 6 вызовов прошли
|
||||
# Убедимся что _SEMAPHORE в модуле имеет нужное значение capacity
|
||||
assert bulk_mod._SEMAPHORE._value >= 0 # семафор сброшен после всех задач
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_http_403_raises_waf_no_retry() -> None:
|
||||
"""403 → NspdBulkWafError немедленно, без retry."""
|
||||
import httpx
|
||||
|
||||
call_count = 0
|
||||
|
||||
async def fake_get(*args: Any, **kwargs: Any) -> httpx.Response:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
return httpx.Response(403, text="Forbidden", request=MagicMock())
|
||||
|
||||
async with NSPDBulkClient() as client:
|
||||
assert client._client is not None
|
||||
with patch.object(client._client, "get", new=AsyncMock(side_effect=fake_get)):
|
||||
with pytest.raises(NspdBulkWafError):
|
||||
await client._get_json("https://nspd.gov.ru/test")
|
||||
|
||||
assert call_count == 1 # ровно 1 попытка, нет retry
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_http_429_backoff_exhausted() -> None:
|
||||
"""429 → exponential backoff 3 retries → NspdBulkRateLimitError."""
|
||||
import httpx
|
||||
|
||||
from app.scrapers.nspd_bulk_client import NspdBulkRateLimitError
|
||||
|
||||
call_count = 0
|
||||
|
||||
async def fake_get(*args: Any, **kwargs: Any) -> httpx.Response:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
return httpx.Response(429, text="Too Many Requests", request=MagicMock())
|
||||
|
||||
# Патчим sleep чтобы тест не ждал 2+4+8 секунд
|
||||
async with NSPDBulkClient(max_retries=3) as client:
|
||||
assert client._client is not None
|
||||
with (
|
||||
patch.object(client._client, "get", new=AsyncMock(side_effect=fake_get)),
|
||||
patch("app.scrapers.nspd_bulk_client.asyncio.sleep", new=AsyncMock()),
|
||||
):
|
||||
with pytest.raises(NspdBulkRateLimitError):
|
||||
await client._get_json("https://nspd.gov.ru/test")
|
||||
|
||||
# 1 оригинальный + 3 retry = 4 вызова
|
||||
assert call_count == 4
|
||||
|
||||
|
||||
# ── E2E tests (network, skipped by default) ───────────────────────────────────
|
||||
|
||||
pytestmark_slow = pytest.mark.slow
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.skip(reason="E2E: требует сеть NSPD — запускать вручную с -m slow")
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_by_quarter_real_ekb() -> None:
|
||||
"""E2E: поиск по реальному кварталу ЕКБ — 66:41:0303161.
|
||||
|
||||
Проверяет:
|
||||
- features > 100 (квартал непустой)
|
||||
- meta содержит ключи 36368/36369/36383 с totalCount > 0
|
||||
- overflow_categories не пуст (значит totalCount > 20 для хотя бы одной)
|
||||
- headers корректны (нет 403/429)
|
||||
"""
|
||||
async with NSPDBulkClient() as client:
|
||||
snap = await client.search_by_quarter("66:41:0303161")
|
||||
|
||||
assert len(snap.features) > 0, "Ожидали хотя бы 1 feature"
|
||||
# Meta должна содержать основные категории
|
||||
assert 36368 in snap.meta_counts, "Нет categoryId 36368 (ЗУ ЕГРН) в meta"
|
||||
assert 36369 in snap.meta_counts, "Нет categoryId 36369 (Здания) в meta"
|
||||
assert snap.meta_counts[36368] > 0, "ЗУ ЕГРН totalCount должен быть > 0"
|
||||
assert snap.meta_counts[36369] > 0, "Здания totalCount должен быть > 0"
|
||||
# В этом квартале >170 ЗУ и >256 зданий → overflow
|
||||
assert len(snap.overflow_categories) > 0, "Ожидали overflow_categories непустой"
|
||||
# total_meta_count > 100 (квартал богатый данными)
|
||||
assert snap.total_meta_count > 100
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.skip(reason="E2E: требует сеть NSPD — запускать вручную с -m slow")
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_by_quarter_empty() -> None:
|
||||
"""E2E: несуществующий квартал → пустой snapshot."""
|
||||
async with NSPDBulkClient() as client:
|
||||
snap = await client.search_by_quarter("66:41:9999999")
|
||||
|
||||
# Должно вернуть пустой snapshot (не raise)
|
||||
assert snap.features == []
|
||||
assert snap.quarter == "66:41:9999999"
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.skip(reason="E2E: требует сеть NSPD — запускать вручную с -m slow")
|
||||
@pytest.mark.asyncio
|
||||
async def test_wms_feature_info_real() -> None:
|
||||
"""E2E: WMS GetFeatureInfo реальный bbox из ЕКБ.
|
||||
|
||||
Использует bbox центра квартала 66:41:0303161 в EPSG:3857.
|
||||
"""
|
||||
# Центральная точка квартала 66:41:0303161 в 3857 (примерно)
|
||||
# ЕКБ центр ≈ 60.6°E, 56.84°N → 3857 ≈ (6748000, 7772000)
|
||||
center_x = 6748000.0
|
||||
center_y = 7772000.0
|
||||
half = 500.0 # 500м bbox
|
||||
bbox = (center_x - half, center_y - half, center_x + half, center_y + half)
|
||||
|
||||
async with NSPDBulkClient() as client:
|
||||
features = await client.wms_feature_info(
|
||||
layer_id=36368, # ЗУ ЕГРН
|
||||
bbox=bbox,
|
||||
click_xy=(256, 256),
|
||||
width=512,
|
||||
height=512,
|
||||
)
|
||||
|
||||
# Хотя бы 1 feature в bbox центра ЕКБ квартала
|
||||
assert len(features) >= 1, f"Ожидали ≥1 feature, получили {len(features)}"
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.skip(reason="E2E: требует сеть NSPD — запускать вручную с -m slow")
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_objects_in_building_real() -> None:
|
||||
"""E2E: реальный objdocId=42065602 → >150 flats_cad_nums."""
|
||||
async with NSPDBulkClient() as client:
|
||||
listing = await client.list_objects_in_building(objdoc_id=42065602)
|
||||
|
||||
assert listing.objdoc_id == 42065602
|
||||
assert listing.flats_count > 150, f"Ожидали >150 помещений, получили {listing.flats_count}"
|
||||
assert (
|
||||
len(listing.flats_cad_nums) > 150
|
||||
), f"Ожидали >150 cad_nums помещений, получили {len(listing.flats_cad_nums)}"
|
||||
Loading…
Add table
Reference in a new issue