feat(scrapers): NSPD client foundation (#94) — search + WMS + layers #98
3 changed files with 773 additions and 2 deletions
|
|
@ -1020,7 +1020,7 @@ def analyze_parcel(
|
|||
|
||||
Issue #93 — Graceful fallback при отсутствии geometry:
|
||||
- Не возвращаем 404 сразу. Вместо: enqueue NSPD on-demand fetch, ждём
|
||||
inline до _INLINE_FETCH_WAIT_S (~25с). Если за это время геометрия
|
||||
inline до _INLINE_FETCH_WAIT_S (~15с). Если за это время геометрия
|
||||
появилась в БД — продолжаем analyze (fast path).
|
||||
- Иначе → 202 Accepted + {status, job_id, eta_seconds} для polling
|
||||
через GET /fetch-status.
|
||||
|
|
@ -1057,7 +1057,8 @@ def analyze_parcel(
|
|||
)
|
||||
|
||||
if not row:
|
||||
# #93 — graceful fallback: enqueue NSPD fetch, await inline до 25s.
|
||||
# #93 — graceful fallback: enqueue NSPD fetch, await inline до 15s
|
||||
# (см. _INLINE_FETCH_WAIT_S — снижено с 25 для threadpool safety).
|
||||
status, job_id, error_msg = find_or_enqueue_fetch(db, cad_num)
|
||||
if status == "invalid_format":
|
||||
raise HTTPException(status_code=400, detail=error_msg)
|
||||
|
|
|
|||
449
backend/app/services/scrapers/nspd_client.py
Normal file
449
backend/app/services/scrapers/nspd_client.py
Normal file
|
|
@ -0,0 +1,449 @@
|
|||
"""Унифицированный NSPD API client — foundation для G1/G2/G3/E1/#93.
|
||||
|
||||
Issue #94: при исследовании НСПД через debug browser найдено что
|
||||
REST `/api/geoportal/v2/search/geoportal?query={cad}` возвращает GeoJSON
|
||||
+ полные properties (ВРИ, категория земель, кадастровая стоимость, адрес)
|
||||
за один запрос. Плюс WMS endpoints для тематических слоёв (ПЗЗ, ЗОУИТ,
|
||||
территориальные зоны, risk zones).
|
||||
|
||||
Этот клиент покрывает 4 базовых метода API:
|
||||
|
||||
- search_by_cad(cad) → SearchResult — поиск по cad-номеру
|
||||
- get_feature_info(layer_id, lon, lat) → list[Feature] — feature data на точке
|
||||
- get_features_in_bbox(layer_id, bbox) → list[Feature] — bbox bulk через
|
||||
GetFeatureInfo (WFS GetCapabilities → 404, workaround)
|
||||
- list_layers(theme_id) → list[Layer] — каталог слоёв в теме
|
||||
|
||||
Reuse: `fetch_geoportal` из `nspd_lite.py` (WAF-compatible urllib + rate limit).
|
||||
Расширения сверх nspd_lite:
|
||||
- Typed wrappers (NSPDFeature, NSPDLayer, NSPDSearchResult)
|
||||
- WMS GetFeatureInfo / GetMap helpers (нет в nspd_lite)
|
||||
- Coordinate transform EPSG:3857 ↔ 4326 (NSPD WMS работает в Mercator)
|
||||
|
||||
WMS endpoints (per #94 issue body, TIER 1-6 каталог слоёв):
|
||||
- TIER 1 критичные: 36048 ЗУ, 36071 кварталы, 36049 здания, 36328 сооружения,
|
||||
875838 территориальные зоны (G1), 879243 красные линии
|
||||
- TIER 2 ЗОУИТ (G3): 37577 ОКН, 37578 энергетики/связи, 37580 природные,
|
||||
37579 охраняемых объектов, 37581 иные
|
||||
- TIER 3 риски: 872202 подтопление, 872205 затопление, 872203 заболачивание,
|
||||
872210 обвально-осыпные, 872211 абразия, etc.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from app.services.scrapers.nspd_lite import (
|
||||
_SSL_CTX,
|
||||
HEADERS,
|
||||
NspdLiteError,
|
||||
NspdLiteWafError,
|
||||
fetch_geoportal,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# WMS base — отдельный путь от geoportal/v2/search
|
||||
NSPD_WMS_BASE = "https://nspd.gov.ru/api/aeggis/v4"
|
||||
# Layers catalog endpoint
|
||||
NSPD_LAYERS_TREE = "https://nspd.gov.ru/api/geoportal/v1/layers-theme-tree"
|
||||
# Themes list
|
||||
NSPD_THEMES = "https://nspd.gov.ru/api/geoportal/v1/layers-theme"
|
||||
|
||||
# Стандартные theme ID
|
||||
THEME_PKK = 1
|
||||
THEME_ARN = 665
|
||||
|
||||
|
||||
# ── Layer ID catalog (TIER 1-6 per #94) ──────────────────────────────────────
|
||||
# Не enum, а module-level dict — для удобства интроспекции и тестов.
|
||||
# Использовать как: LAYERS["territorial_zones"] = 875838.
|
||||
|
||||
LAYERS: dict[str, int] = {
|
||||
# TIER 1 — критичные для МКД (Gate-факторы)
|
||||
"parcels": 36048, # Земельные участки ЕГРН
|
||||
"quarters": 36071, # Кадастровые кварталы
|
||||
"buildings": 36049, # Здания ЕГРН
|
||||
"engineering_structures": 36328, # Сооружения ЕГРН
|
||||
"territorial_zones": 875838, # ПЗЗ (G1 #28)
|
||||
"red_lines": 879243, # Красные линии
|
||||
# TIER 2 — ЗОУИТ (G3 #30)
|
||||
"zouit_okn": 37577, # ОКН
|
||||
"zouit_engineering": 37578, # энергетики/связи/транспорта
|
||||
"zouit_natural": 37580, # природные (водоохранные, СЗО)
|
||||
"zouit_protected": 37579, # охраняемых объектов (СЗЗ предприятий)
|
||||
"zouit_other": 37581, # иные (публичный сервитут, ОЭЗ)
|
||||
# TIER 3 — Risk / Negative
|
||||
"risk_flooding_underground": 872202, # подтопление
|
||||
"risk_flooding": 872205, # затопление
|
||||
"risk_swampification": 872203, # заболачивание
|
||||
"risk_landslide": 872210, # обвально-осыпные
|
||||
"risk_abrasion": 872211, # абразия
|
||||
"risk_erosion_water": 872153, # водная эрозия
|
||||
"risk_erosion_linear": 872155,
|
||||
"risk_erosion_wind": 872164,
|
||||
"risk_desertification": 872182,
|
||||
"risk_clutter": 872206, # захламление
|
||||
"risk_burns": 872222, # гари
|
||||
# TIER 4 — Opportunity / nature
|
||||
"protected_areas": 875845, # ООПТ
|
||||
"coast_lines_river": 875832,
|
||||
"coast_lines_sea": 875835,
|
||||
"forestries": 875866,
|
||||
"auction_parcels": 37299, # ЗУ на аукционе — opportunity
|
||||
"scheme_parcels": 37294, # ЗУ по схеме расположения
|
||||
"free_parcels": 37298, # ЗУ свободные от прав
|
||||
"future_parcels": 36473, # ЗУ по проекту межевания
|
||||
"okn_territory": 875840,
|
||||
"cadastral_cost_heatmap": 37236,
|
||||
"cadastral_cost_per_m2_heatmap": 37758,
|
||||
}
|
||||
|
||||
|
||||
# Default rate limit (мс между запросами) — баланс между скоростью и WAF
|
||||
DEFAULT_RATE_MS = 600
|
||||
|
||||
# SSL context — reuse `_SSL_CTX` из nspd_lite (one source of truth для WAF-
|
||||
# compatible TLS settings; нет дублирования если что-то поменяется).
|
||||
|
||||
|
||||
# ── Typed responses ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NSPDFeature:
|
||||
"""Парсенный GeoJSON Feature из NSPD API.
|
||||
|
||||
`geometry` — raw GeoJSON dict (Polygon/LineString/Point в EPSG:3857).
|
||||
`properties` — все returned attributes (cad_num, ВРИ, кост и т.д.).
|
||||
"""
|
||||
|
||||
feature_id: str | None
|
||||
geometry: dict[str, Any] | None
|
||||
properties: dict[str, Any]
|
||||
crs: str = "EPSG:3857" # NSPD default; для search /v2 фактически 3857
|
||||
|
||||
@classmethod
|
||||
def from_raw(cls, raw: dict[str, Any]) -> NSPDFeature:
|
||||
"""Парсим GeoJSON Feature dict в typed NSPDFeature."""
|
||||
props = raw.get("properties") or {}
|
||||
return cls(
|
||||
feature_id=str(raw.get("id")) if raw.get("id") is not None else None,
|
||||
geometry=raw.get("geometry"),
|
||||
properties=props,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NSPDSearchResult:
|
||||
"""Aggregated результат поиска по cad — typed wrapper над feature list."""
|
||||
|
||||
cad_num: str
|
||||
features: list[NSPDFeature]
|
||||
raw: dict[str, Any] # original response для debugging / future-proofing
|
||||
|
||||
@property
|
||||
def first(self) -> NSPDFeature | None:
|
||||
"""Удобный shortcut: features[0] или None."""
|
||||
return self.features[0] if self.features else None
|
||||
|
||||
@property
|
||||
def is_empty(self) -> bool:
|
||||
return not self.features
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NSPDLayer:
|
||||
"""Метаданные layer'а из layers-theme-tree response."""
|
||||
|
||||
layer_id: int
|
||||
title: str
|
||||
layer_type: str | None # 'wms' | 'wfs' | etc
|
||||
metadata: dict[str, Any]
|
||||
|
||||
|
||||
# ── HTTP helper ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _http_get_json(url: str, *, timeout: int = 15, rate_ms: int = 0) -> Any:
|
||||
"""Внутренний GET-JSON helper с WAF-handling, paralleling nspd_lite.
|
||||
|
||||
Используется для endpoints которые не покрыты `fetch_geoportal` (WMS,
|
||||
layers-tree). Reuse HEADERS + SSL context из nspd_lite — общая защита от
|
||||
WAF блокировки по TLS fingerprint.
|
||||
"""
|
||||
if rate_ms > 0:
|
||||
time.sleep(rate_ms / 1000.0)
|
||||
req = urllib.request.Request(url, headers=HEADERS)
|
||||
try:
|
||||
with urllib.request.urlopen(req, context=_SSL_CTX, timeout=timeout) as r:
|
||||
body = r.read().decode("utf-8", "ignore")
|
||||
return json.loads(body)
|
||||
except urllib.error.HTTPError as e:
|
||||
body = e.read().decode("utf-8", "ignore")[:300] if e.fp else ""
|
||||
if e.code in (403, 429):
|
||||
raise NspdLiteWafError(f"HTTP {e.code} (WAF/rate-limit): {body}") from e
|
||||
raise NspdLiteError(f"HTTP {e.code}: {body}") from e
|
||||
except urllib.error.URLError as e:
|
||||
raise NspdLiteError(f"Network error: {e}") from e
|
||||
|
||||
|
||||
# ── Coordinate transforms ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def lonlat_to_3857(lon: float, lat: float) -> tuple[float, float]:
|
||||
"""WGS84 lon/lat → EPSG:3857 Web Mercator (метры).
|
||||
|
||||
Стандартная формула; стандартная для всех Web mapping. NSPD WMS принимает
|
||||
bbox в 3857.
|
||||
"""
|
||||
x = lon * 20037508.34 / 180.0
|
||||
y = math.log(math.tan((90.0 + lat) * math.pi / 360.0)) / (math.pi / 180.0)
|
||||
y = y * 20037508.34 / 180.0
|
||||
return (x, y)
|
||||
|
||||
|
||||
def bbox_around_point_m(
|
||||
lon: float,
|
||||
lat: float,
|
||||
buffer_m: int,
|
||||
) -> tuple[float, float, float, float]:
|
||||
"""Микро-bbox вокруг точки в EPSG:3857 — для WMS GetFeatureInfo.
|
||||
|
||||
Returns (xmin, ymin, xmax, ymax). buffer_m по m в каждую сторону.
|
||||
"""
|
||||
x, y = lonlat_to_3857(lon, lat)
|
||||
return (x - buffer_m, y - buffer_m, x + buffer_m, y + buffer_m)
|
||||
|
||||
|
||||
# ── Public client API ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class NSPDClient:
|
||||
"""Foundation-level NSPD client. 4 core methods.
|
||||
|
||||
Stateless — все методы могут вызываться независимо. Используй один instance
|
||||
на process (rate-limit shared между вызовами через time.sleep).
|
||||
"""
|
||||
|
||||
def __init__(self, rate_ms: int = DEFAULT_RATE_MS, timeout: int = 15) -> None:
|
||||
self.rate_ms = rate_ms
|
||||
self.timeout = timeout
|
||||
|
||||
# ── 1. search_by_cad ────────────────────────────────────────────────────
|
||||
|
||||
def search_by_cad(self, cad_num: str, thematic_id: int = 1) -> NSPDSearchResult:
|
||||
"""REST поиск /v2/search/geoportal — 1 запрос = geometry + ВРИ + кост.
|
||||
|
||||
Args:
|
||||
cad_num: канонический формат — `NN:NN:NNNNNN[:NN[:NN]]`, цифры через `:`.
|
||||
Примеры:
|
||||
- 3-сегментный квартал: `'66:41:0204016'`
|
||||
- 4-сегментный участок: `'66:41:0204016:10'`
|
||||
- 5-сегментное здание: `'66:41:0204016:10:1'`
|
||||
Спецсимволы / пробелы корректно URL-encode'нутся (нет SQL/URL
|
||||
injection), но НСПД молча вернёт empty result. Caller обязан
|
||||
сам валидировать формат если нужна defensive обработка (см.
|
||||
`app.services.site_finder.cadastre_fetch.validate_cad_format`).
|
||||
thematic_id: 1 ЗУ / 2 квартал / 5 здание / 7 территориальная зона.
|
||||
Default 1 — соответствует use-case G2 ВРИ.
|
||||
|
||||
Returns:
|
||||
NSPDSearchResult с typed features. `result.is_empty` → cad не найден.
|
||||
|
||||
Raises:
|
||||
NspdLiteWafError при 403/429 — caller должен сделать backoff.
|
||||
NspdLiteError при прочих 4xx/5xx.
|
||||
|
||||
Закрывает: G2 #29 ВРИ, поддержка on-demand #93/#51.
|
||||
"""
|
||||
raw = fetch_geoportal(
|
||||
cad_num,
|
||||
thematic_id=thematic_id,
|
||||
timeout=self.timeout,
|
||||
rate_ms=self.rate_ms,
|
||||
)
|
||||
# Response shape: {"data": {"type": "FeatureCollection", "features": [...]}}
|
||||
data = raw.get("data") or {}
|
||||
feats = data.get("features") or []
|
||||
features = [NSPDFeature.from_raw(f) for f in feats]
|
||||
return NSPDSearchResult(cad_num=cad_num, features=features, raw=raw)
|
||||
|
||||
# ── 2. get_feature_info ─────────────────────────────────────────────────
|
||||
|
||||
def get_feature_info(
|
||||
self,
|
||||
layer_id: int,
|
||||
lon: float,
|
||||
lat: float,
|
||||
buffer_m: int = 100,
|
||||
) -> list[NSPDFeature]:
|
||||
"""WMS GetFeatureInfo на точке — какие feature'ы layer'а покрывают (lon, lat).
|
||||
|
||||
Использует микро-bbox вокруг точки в EPSG:3857 (WMS требует bbox + I/J
|
||||
пиксель). Tile 256x256, I=128, J=128 — центр.
|
||||
|
||||
Returns features (может быть 0+).
|
||||
|
||||
Use cases:
|
||||
- ВРИ конкретного участка (layer_id=875838 territorial_zones)
|
||||
- ЗОУИТ на участке (layer_id=37577..37581)
|
||||
- Risk zones overlap (872xxx)
|
||||
"""
|
||||
xmin, ymin, xmax, ymax = bbox_around_point_m(lon, lat, buffer_m)
|
||||
params = {
|
||||
"SERVICE": "WMS",
|
||||
"VERSION": "1.3.0",
|
||||
"REQUEST": "GetFeatureInfo",
|
||||
"FORMAT": "image/png",
|
||||
"TRANSPARENT": "true",
|
||||
"QUERY_LAYERS": str(layer_id),
|
||||
"LAYERS": str(layer_id),
|
||||
"INFO_FORMAT": "application/json",
|
||||
# STYLES обязателен (пустой), иначе INFO_FORMAT игнорируется per NSPD WAF
|
||||
"STYLES": "",
|
||||
"I": "128",
|
||||
"J": "128",
|
||||
"WIDTH": "256",
|
||||
"HEIGHT": "256",
|
||||
"CRS": "EPSG:3857",
|
||||
"BBOX": f"{xmin},{ymin},{xmax},{ymax}",
|
||||
}
|
||||
url = f"{NSPD_WMS_BASE}/{layer_id}/wms?{urllib.parse.urlencode(params)}"
|
||||
data = _http_get_json(url, timeout=self.timeout, rate_ms=self.rate_ms)
|
||||
feats = (data or {}).get("features") or []
|
||||
return [NSPDFeature.from_raw(f) for f in feats]
|
||||
|
||||
# ── 3. get_features_in_bbox ─────────────────────────────────────────────
|
||||
|
||||
def get_features_in_bbox(
|
||||
self,
|
||||
layer_id: int,
|
||||
bbox_3857: tuple[float, float, float, float],
|
||||
*,
|
||||
width: int = 4096,
|
||||
height: int = 4096,
|
||||
) -> list[NSPDFeature]:
|
||||
"""Bulk fetch features в bbox через GetFeatureInfo с большим bbox.
|
||||
|
||||
Workaround: WFS GetCapabilities → 404 на nspd.gov.ru, нет WFS
|
||||
GetFeature endpoint. Решение: использовать GetFeatureInfo с large
|
||||
bbox и точкой в центре (I=W/2, J=H/2) — возвращает все features
|
||||
пересекающиеся с bbox.
|
||||
|
||||
Args:
|
||||
bbox_3857: (xmin, ymin, xmax, ymax) в EPSG:3857 метрах.
|
||||
width/height: размер виртуального tile. Большой → большой bbox.
|
||||
|
||||
Returns:
|
||||
list[NSPDFeature]; пусто если ничего не найдено.
|
||||
|
||||
Use cases (per #94 acceptance):
|
||||
- sync_territorial_zones_bbox → закрывает G1 #28 ПЗЗ
|
||||
- sync_zouit_*_bbox → G3 #30
|
||||
- sync_risk_zones_bbox → новый risk overlay
|
||||
"""
|
||||
xmin, ymin, xmax, ymax = bbox_3857
|
||||
params = {
|
||||
"SERVICE": "WMS",
|
||||
"VERSION": "1.3.0",
|
||||
"REQUEST": "GetFeatureInfo",
|
||||
"FORMAT": "image/png",
|
||||
"TRANSPARENT": "true",
|
||||
"QUERY_LAYERS": str(layer_id),
|
||||
"LAYERS": str(layer_id),
|
||||
"INFO_FORMAT": "application/json",
|
||||
"STYLES": "",
|
||||
"I": str(width // 2),
|
||||
"J": str(height // 2),
|
||||
"WIDTH": str(width),
|
||||
"HEIGHT": str(height),
|
||||
"CRS": "EPSG:3857",
|
||||
"BBOX": f"{xmin},{ymin},{xmax},{ymax}",
|
||||
}
|
||||
url = f"{NSPD_WMS_BASE}/{layer_id}/wms?{urllib.parse.urlencode(params)}"
|
||||
data = _http_get_json(url, timeout=self.timeout, rate_ms=self.rate_ms)
|
||||
feats = (data or {}).get("features") or []
|
||||
return [NSPDFeature.from_raw(f) for f in feats]
|
||||
|
||||
# ── 4. list_layers ──────────────────────────────────────────────────────
|
||||
|
||||
def list_layers(self, theme_id: int = THEME_PKK) -> list[NSPDLayer]:
|
||||
"""Полный каталог слоёв в теме (PKK / ARN).
|
||||
|
||||
Use case: admin UI «выбрать layer для bulk-импорта», discovery новых
|
||||
слоёв при изменении NSPD каталога.
|
||||
"""
|
||||
url = f"{NSPD_LAYERS_TREE}?themeId={theme_id}"
|
||||
data = _http_get_json(url, timeout=self.timeout, rate_ms=self.rate_ms)
|
||||
# Shape defense: NSPD иногда оборачивает в {"data": [...]} — паттерн уже
|
||||
# ловили в Bug_Nspd_Geo_Str_Object_No_Get_Fixed. Без проверки walker
|
||||
# молча вернул бы [], а caller увидел бы пустой catalog.
|
||||
if isinstance(data, dict) and "data" in data and not data.get("children"):
|
||||
data = data.get("data") or []
|
||||
if not isinstance(data, list | dict):
|
||||
logger.warning(
|
||||
"list_layers: unexpected NSPD response type %s for theme_id=%s",
|
||||
type(data).__name__,
|
||||
theme_id,
|
||||
)
|
||||
return []
|
||||
layers: list[NSPDLayer] = []
|
||||
# Tree response: nested {"children": [...], "id": N, "title": "..."}
|
||||
# Используем walker по subtree чтобы вытащить все leaf layers
|
||||
for entry in _walk_layer_tree(data):
|
||||
layers.append(
|
||||
NSPDLayer(
|
||||
layer_id=int(entry["id"]),
|
||||
title=str(entry.get("title", "")),
|
||||
layer_type=entry.get("layerType") or entry.get("layer_type"),
|
||||
metadata=entry,
|
||||
)
|
||||
)
|
||||
return layers
|
||||
|
||||
|
||||
def _walk_layer_tree(node: Any) -> list[dict[str, Any]]:
|
||||
"""Рекурсивный walker для NSPD layers-theme-tree. Yields leaf nodes."""
|
||||
out: list[dict[str, Any]] = []
|
||||
if isinstance(node, dict):
|
||||
# Leaf если есть id + нет children (или children пустой)
|
||||
children = node.get("children") or []
|
||||
if "id" in node and not children:
|
||||
out.append(node)
|
||||
elif "id" in node and children:
|
||||
# Branch с id — может быть и сам layer, и группа. NSPD сейчас
|
||||
# treats group nodes как "не имеющий wms-endpoint" — но включаем
|
||||
# на случай если они оба leaves (хитрая branch).
|
||||
out.append(node)
|
||||
for ch in children:
|
||||
out.extend(_walk_layer_tree(ch))
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
out.extend(_walk_layer_tree(item))
|
||||
return out
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LAYERS",
|
||||
"NSPD_THEMES",
|
||||
"NSPD_WMS_BASE",
|
||||
"THEME_ARN",
|
||||
"THEME_PKK",
|
||||
"NSPDClient",
|
||||
"NSPDFeature",
|
||||
"NSPDLayer",
|
||||
"NSPDSearchResult",
|
||||
"NspdLiteError",
|
||||
"NspdLiteWafError",
|
||||
"bbox_around_point_m",
|
||||
"lonlat_to_3857",
|
||||
]
|
||||
321
backend/tests/test_nspd_client.py
Normal file
321
backend/tests/test_nspd_client.py
Normal file
|
|
@ -0,0 +1,321 @@
|
|||
"""Unit tests для nspd_client (#94). Без сетевых запросов — только parsing.
|
||||
|
||||
Network calls (search_by_cad / get_feature_info / list_layers) проверяются
|
||||
вручную через `python -c "from app.services.scrapers.nspd_client import
|
||||
NSPDClient; print(NSPDClient().search_by_cad('66:41:0204016:10'))"` —
|
||||
проверяется в production smoke test после deploy.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.nspd_client import (
|
||||
LAYERS,
|
||||
NSPDClient,
|
||||
NSPDFeature,
|
||||
NSPDLayer,
|
||||
NSPDSearchResult,
|
||||
_walk_layer_tree,
|
||||
bbox_around_point_m,
|
||||
lonlat_to_3857,
|
||||
)
|
||||
|
||||
|
||||
def test_layers_catalog_has_critical_ids() -> None:
|
||||
"""Sanity check: TIER 1 critical layers зарегистрированы."""
|
||||
# Из #94: TIER 1 включает 875838 (территориальные зоны / ПЗЗ G1)
|
||||
assert LAYERS["territorial_zones"] == 875838
|
||||
assert LAYERS["parcels"] == 36048
|
||||
assert LAYERS["buildings"] == 36049
|
||||
assert LAYERS["quarters"] == 36071
|
||||
# TIER 2 ЗОУИТ (G3)
|
||||
assert LAYERS["zouit_okn"] == 37577
|
||||
assert LAYERS["zouit_engineering"] == 37578
|
||||
assert LAYERS["zouit_natural"] == 37580
|
||||
|
||||
|
||||
def test_lonlat_to_3857_zero() -> None:
|
||||
"""0,0 → ~0,0 (with tiny floating error)."""
|
||||
x, y = lonlat_to_3857(0.0, 0.0)
|
||||
assert abs(x) < 1.0
|
||||
assert abs(y) < 1.0
|
||||
|
||||
|
||||
def test_lonlat_to_3857_ekb_center() -> None:
|
||||
"""ЕКБ центр (60.605, 56.838) — known approximate projection."""
|
||||
x, y = lonlat_to_3857(60.605, 56.838)
|
||||
# Sanity: положительный x (восточная долгота), положительный y (северная)
|
||||
assert x > 6_700_000
|
||||
assert x < 6_800_000
|
||||
assert y > 7_700_000
|
||||
assert y < 7_800_000
|
||||
|
||||
|
||||
def test_bbox_around_point_m_basic() -> None:
|
||||
"""100m buffer вокруг точки → bbox с side ≈ 200m."""
|
||||
xmin, ymin, xmax, ymax = bbox_around_point_m(60.605, 56.838, 100)
|
||||
assert abs((xmax - xmin) - 200) < 1.0
|
||||
assert abs((ymax - ymin) - 200) < 1.0
|
||||
|
||||
|
||||
def test_nspd_feature_from_raw_minimal() -> None:
|
||||
"""Парсинг минимального GeoJSON Feature."""
|
||||
raw = {
|
||||
"id": "feat-1",
|
||||
"geometry": {"type": "Point", "coordinates": [1, 2]},
|
||||
"properties": {"cad_num": "66:41:0204016:10"},
|
||||
}
|
||||
feat = NSPDFeature.from_raw(raw)
|
||||
assert feat.feature_id == "feat-1"
|
||||
assert feat.geometry is not None
|
||||
assert feat.geometry["type"] == "Point"
|
||||
assert feat.properties["cad_num"] == "66:41:0204016:10"
|
||||
assert feat.crs == "EPSG:3857"
|
||||
|
||||
|
||||
def test_nspd_feature_from_raw_missing_id() -> None:
|
||||
"""Feature без id → feature_id is None."""
|
||||
feat = NSPDFeature.from_raw({"geometry": None, "properties": {}})
|
||||
assert feat.feature_id is None
|
||||
assert feat.geometry is None
|
||||
assert feat.properties == {}
|
||||
|
||||
|
||||
def test_nspd_search_result_empty() -> None:
|
||||
"""Empty result → first is None, is_empty True."""
|
||||
result = NSPDSearchResult(cad_num="x", features=[], raw={})
|
||||
assert result.first is None
|
||||
assert result.is_empty is True
|
||||
|
||||
|
||||
def test_nspd_search_result_nonempty() -> None:
|
||||
"""First feature accessible через .first."""
|
||||
feat = NSPDFeature.from_raw({"id": "x", "geometry": None, "properties": {}})
|
||||
result = NSPDSearchResult(cad_num="x", features=[feat], raw={})
|
||||
assert result.first is feat
|
||||
assert result.is_empty is False
|
||||
|
||||
|
||||
def test_walk_layer_tree_flat() -> None:
|
||||
"""Flat list of layers — все попадают в output."""
|
||||
tree: list[dict[str, Any]] = [
|
||||
{"id": 1, "title": "A", "children": []},
|
||||
{"id": 2, "title": "B", "children": []},
|
||||
]
|
||||
out = _walk_layer_tree(tree)
|
||||
ids = sorted(x["id"] for x in out)
|
||||
assert ids == [1, 2]
|
||||
|
||||
|
||||
def test_walk_layer_tree_nested() -> None:
|
||||
"""Nested tree — все leaves вытаскиваются."""
|
||||
tree = {
|
||||
"id": 1,
|
||||
"title": "root",
|
||||
"children": [
|
||||
{"id": 2, "title": "child", "children": []},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "branch",
|
||||
"children": [
|
||||
{"id": 4, "title": "grandchild", "children": []},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
out = _walk_layer_tree(tree)
|
||||
ids = sorted(x["id"] for x in out)
|
||||
# Все уровни включены — root, child, branch, grandchild
|
||||
assert ids == [1, 2, 3, 4]
|
||||
|
||||
|
||||
def test_walk_layer_tree_empty() -> None:
|
||||
"""Empty input → empty output, no crash."""
|
||||
assert _walk_layer_tree({}) == []
|
||||
assert _walk_layer_tree([]) == []
|
||||
assert _walk_layer_tree(None) == [] # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_nspd_client_search_by_cad_parses_response(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""search_by_cad оборачивает fetch_geoportal в NSPDSearchResult."""
|
||||
fake_raw = {
|
||||
"data": {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"id": "f1",
|
||||
"geometry": {"type": "Polygon", "coordinates": []},
|
||||
"properties": {
|
||||
"cad_num": "66:41:0204016:10",
|
||||
"land_record_area": 1500,
|
||||
"permitted_use_established_by_document": "Многоэтажная жилая",
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
def fake_fetch(query: str, **kwargs: Any) -> dict[str, Any]:
|
||||
assert query == "66:41:0204016:10"
|
||||
return fake_raw
|
||||
|
||||
# Patch the imported reference in nspd_client (it was imported by name)
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.nspd_client.fetch_geoportal",
|
||||
fake_fetch,
|
||||
)
|
||||
|
||||
client = NSPDClient()
|
||||
result = client.search_by_cad("66:41:0204016:10")
|
||||
assert result.cad_num == "66:41:0204016:10"
|
||||
assert len(result.features) == 1
|
||||
assert result.first is not None
|
||||
assert result.first.properties["permitted_use_established_by_document"] == "Многоэтажная жилая"
|
||||
assert result.raw is fake_raw # preserved для debugging
|
||||
|
||||
|
||||
def test_nspd_client_search_by_cad_empty(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Empty response → is_empty True, no crash."""
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.nspd_client.fetch_geoportal",
|
||||
lambda *a, **kw: {"data": {"type": "FeatureCollection", "features": []}},
|
||||
)
|
||||
result = NSPDClient().search_by_cad("99:99:9999999")
|
||||
assert result.is_empty is True
|
||||
assert result.first is None
|
||||
|
||||
|
||||
def test_nspd_layer_dataclass() -> None:
|
||||
"""NSPDLayer immutable, hashable (frozen + slots)."""
|
||||
layer = NSPDLayer(layer_id=875838, title="Терр.зоны", layer_type="wms", metadata={})
|
||||
assert layer.layer_id == 875838
|
||||
# Frozen
|
||||
with pytest.raises(AttributeError):
|
||||
layer.title = "x" # type: ignore[misc]
|
||||
|
||||
|
||||
# ── Mock tests for WMS/layers methods (no network) ──────────────────────────
|
||||
|
||||
|
||||
def test_get_feature_info_parses_response(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""get_feature_info вызывает _http_get_json с правильным URL + парсит features."""
|
||||
captured_urls: list[str] = []
|
||||
|
||||
def fake_http(url: str, **kwargs: Any) -> dict[str, Any]:
|
||||
captured_urls.append(url)
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"id": "tz-1",
|
||||
"geometry": {"type": "Polygon", "coordinates": []},
|
||||
"properties": {"zone_code": "Ж-1"},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
monkeypatch.setattr("app.services.scrapers.nspd_client._http_get_json", fake_http)
|
||||
client = NSPDClient()
|
||||
feats = client.get_feature_info(875838, lon=60.605, lat=56.838, buffer_m=100)
|
||||
assert len(feats) == 1
|
||||
assert feats[0].properties["zone_code"] == "Ж-1"
|
||||
# Verify URL structure
|
||||
assert len(captured_urls) == 1
|
||||
url = captured_urls[0]
|
||||
assert "/api/aeggis/v4/875838/wms?" in url
|
||||
assert "REQUEST=GetFeatureInfo" in url
|
||||
assert "INFO_FORMAT=application%2Fjson" in url
|
||||
assert "STYLES=" in url # required per NSPD WAF
|
||||
assert "CRS=EPSG%3A3857" in url
|
||||
# bbox should reflect 100m buffer (200m side)
|
||||
assert "BBOX=" in url
|
||||
|
||||
|
||||
def test_get_features_in_bbox_url_builds_correctly(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""get_features_in_bbox использует переданный bbox без преобразования."""
|
||||
captured_urls: list[str] = []
|
||||
|
||||
def fake_http(url: str, **kwargs: Any) -> dict[str, Any]:
|
||||
captured_urls.append(url)
|
||||
return {"features": []}
|
||||
|
||||
monkeypatch.setattr("app.services.scrapers.nspd_client._http_get_json", fake_http)
|
||||
client = NSPDClient()
|
||||
bbox = (6700000.0, 7700000.0, 6800000.0, 7800000.0) # ЕКБ region in 3857
|
||||
feats = client.get_features_in_bbox(875838, bbox)
|
||||
assert feats == []
|
||||
assert len(captured_urls) == 1
|
||||
url = captured_urls[0]
|
||||
# bbox passed directly into BBOX param (URL-encoded comma = %2C)
|
||||
assert "BBOX=6700000.0%2C7700000.0%2C6800000.0%2C7800000.0" in url
|
||||
# Large width/height для bulk fetch
|
||||
assert "WIDTH=4096" in url
|
||||
assert "HEIGHT=4096" in url
|
||||
# I/J pointer в центр виртуального тайла
|
||||
assert "I=2048" in url
|
||||
assert "J=2048" in url
|
||||
|
||||
|
||||
def test_list_layers_walks_tree_response(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""list_layers вызывает /layers-theme-tree и распаковывает в NSPDLayer."""
|
||||
fake_tree = [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "PKK тема",
|
||||
"children": [
|
||||
{
|
||||
"id": 875838,
|
||||
"title": "Территориальные зоны",
|
||||
"layerType": "wms",
|
||||
"children": [],
|
||||
},
|
||||
{
|
||||
"id": 37577,
|
||||
"title": "ЗОУИТ ОКН",
|
||||
"layerType": "wms",
|
||||
"children": [],
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.nspd_client._http_get_json",
|
||||
lambda url, **kw: fake_tree,
|
||||
)
|
||||
client = NSPDClient()
|
||||
layers = client.list_layers(theme_id=1)
|
||||
layer_ids = [layer.layer_id for layer in layers]
|
||||
# Root + 2 children = 3 leaves
|
||||
assert 875838 in layer_ids
|
||||
assert 37577 in layer_ids
|
||||
# Sample layer metadata preserved
|
||||
tz_layer = next(layer for layer in layers if layer.layer_id == 875838)
|
||||
assert tz_layer.title == "Территориальные зоны"
|
||||
assert tz_layer.layer_type == "wms"
|
||||
|
||||
|
||||
def test_list_layers_handles_data_wrapper(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Shape defense — NSPD иногда оборачивает в {"data": [...]}."""
|
||||
fake_wrapped = {
|
||||
"data": [{"id": 36048, "title": "ЗУ", "children": []}],
|
||||
"meta": {"count": 1},
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.nspd_client._http_get_json",
|
||||
lambda url, **kw: fake_wrapped,
|
||||
)
|
||||
layers = NSPDClient().list_layers(theme_id=1)
|
||||
assert len(layers) >= 1
|
||||
assert any(layer.layer_id == 36048 for layer in layers)
|
||||
|
||||
|
||||
def test_list_layers_handles_garbage_response(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Не dict/list response → empty list + warning log (no crash)."""
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.nspd_client._http_get_json",
|
||||
lambda url, **kw: "garbage string",
|
||||
)
|
||||
assert NSPDClient().list_layers(theme_id=1) == []
|
||||
Loading…
Add table
Reference in a new issue