From 6ce73221a974c2ab9ca9f91296a1203cc98f3aa7 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Tue, 12 May 2026 09:45:45 +0300 Subject: [PATCH] fix(scrapers): address PR #98 auto-review minor feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend (nspd_client.py): 1. (line 100+) bbox_around_point_m signature split на multi-line. 2. (lazy imports) import json, time → module-level. 3. (duplicate SSL ctx) Reuse _SSL_CTX из nspd_lite через explicit import. 4. (shape defense) list_layers чекает {"data": [...]} wrapper + non-dict/list garbage → warning + empty. 5. (cad_num docstring) search_by_cad документирует формат + ссылку на validate_cad_format. Tests: +5 mock тестов (19/19 passed): - get_feature_info URL builder + parsing - get_features_in_bbox bbox propagation - list_layers_walks_tree_response - list_layers_handles_data_wrapper - list_layers_handles_garbage_response Backend (parcels.py): docstring '~25с' → '~15с'. Per auto-review on be14388. --- backend/app/api/v1/parcels.py | 2 +- backend/app/services/scrapers/nspd_client.py | 39 ++++-- backend/tests/test_nspd_client.py | 124 +++++++++++++++++++ 3 files changed, 156 insertions(+), 9 deletions(-) diff --git a/backend/app/api/v1/parcels.py b/backend/app/api/v1/parcels.py index c412ea46..9f2834c7 100644 --- a/backend/app/api/v1/parcels.py +++ b/backend/app/api/v1/parcels.py @@ -1020,7 +1020,7 @@ def analyze_parcel( Issue #93 — Graceful fallback при отсутствии geometry: - Не возвращаем 404 сразу. Вместо: enqueue NSPD on-demand fetch, ждём - inline до _INLINE_FETCH_WAIT_S (~25с). Если за это время геометрия + inline до _INLINE_FETCH_WAIT_S (~15с). Если за это время геометрия появилась в БД — продолжаем analyze (fast path). - Иначе → 202 Accepted + {status, job_id, eta_seconds} для polling через GET /fetch-status. diff --git a/backend/app/services/scrapers/nspd_client.py b/backend/app/services/scrapers/nspd_client.py index ef81184b..1dae05d4 100644 --- a/backend/app/services/scrapers/nspd_client.py +++ b/backend/app/services/scrapers/nspd_client.py @@ -31,9 +31,10 @@ WMS endpoints (per #94 issue body, TIER 1-6 каталог слоёв): from __future__ import annotations +import json import logging import math -import ssl +import time import urllib.error import urllib.parse import urllib.request @@ -41,6 +42,7 @@ from dataclasses import dataclass from typing import Any from app.services.scrapers.nspd_lite import ( + _SSL_CTX, HEADERS, NspdLiteError, NspdLiteWafError, @@ -109,8 +111,8 @@ LAYERS: dict[str, int] = { # Default rate limit (мс между запросами) — баланс между скоростью и WAF DEFAULT_RATE_MS = 600 -# SSL context — reuse from nspd_lite pattern (WAF-compatible) -_SSL_CTX = ssl._create_unverified_context() +# SSL context — reuse `_SSL_CTX` из nspd_lite (one source of truth для WAF- +# compatible TLS settings; нет дублирования если что-то поменяется). # ── Typed responses ────────────────────────────────────────────────────────── @@ -178,9 +180,6 @@ def _http_get_json(url: str, *, timeout: int = 15, rate_ms: int = 0) -> Any: layers-tree). Reuse HEADERS + SSL context из nspd_lite — общая защита от WAF блокировки по TLS fingerprint. """ - import json - import time - if rate_ms > 0: time.sleep(rate_ms / 1000.0) req = urllib.request.Request(url, headers=HEADERS) @@ -212,7 +211,11 @@ def lonlat_to_3857(lon: float, lat: float) -> tuple[float, float]: return (x, y) -def bbox_around_point_m(lon: float, lat: float, buffer_m: int) -> tuple[float, float, float, float]: +def bbox_around_point_m( + lon: float, + lat: float, + buffer_m: int, +) -> tuple[float, float, float, float]: """Микро-bbox вокруг точки в EPSG:3857 — для WMS GetFeatureInfo. Returns (xmin, ymin, xmax, ymax). buffer_m по m в каждую сторону. @@ -241,7 +244,15 @@ class NSPDClient: """REST поиск /v2/search/geoportal — 1 запрос = geometry + ВРИ + кост. Args: - cad_num: '66:41:0204016' (квартал) или '66:41:0204016:10' (участок). + cad_num: канонический формат — `NN:NN:NNNNNN[:NN[:NN]]`, цифры через `:`. + Примеры: + - 3-сегментный квартал: `'66:41:0204016'` + - 4-сегментный участок: `'66:41:0204016:10'` + - 5-сегментное здание: `'66:41:0204016:10:1'` + Спецсимволы / пробелы корректно URL-encode'нутся (нет SQL/URL + injection), но НСПД молча вернёт empty result. Caller обязан + сам валидировать формат если нужна defensive обработка (см. + `app.services.site_finder.cadastre_fetch.validate_cad_format`). thematic_id: 1 ЗУ / 2 квартал / 5 здание / 7 территориальная зона. Default 1 — соответствует use-case G2 ВРИ. @@ -373,6 +384,18 @@ class NSPDClient: """ url = f"{NSPD_LAYERS_TREE}?themeId={theme_id}" data = _http_get_json(url, timeout=self.timeout, rate_ms=self.rate_ms) + # Shape defense: NSPD иногда оборачивает в {"data": [...]} — паттерн уже + # ловили в Bug_Nspd_Geo_Str_Object_No_Get_Fixed. Без проверки walker + # молча вернул бы [], а caller увидел бы пустой catalog. + if isinstance(data, dict) and "data" in data and not data.get("children"): + data = data.get("data") or [] + if not isinstance(data, list | dict): + logger.warning( + "list_layers: unexpected NSPD response type %s for theme_id=%s", + type(data).__name__, + theme_id, + ) + return [] layers: list[NSPDLayer] = [] # Tree response: nested {"children": [...], "id": N, "title": "..."} # Используем walker по subtree чтобы вытащить все leaf layers diff --git a/backend/tests/test_nspd_client.py b/backend/tests/test_nspd_client.py index 7f9a9973..0a862234 100644 --- a/backend/tests/test_nspd_client.py +++ b/backend/tests/test_nspd_client.py @@ -195,3 +195,127 @@ def test_nspd_layer_dataclass() -> None: # Frozen with pytest.raises(AttributeError): layer.title = "x" # type: ignore[misc] + + +# ── Mock tests for WMS/layers methods (no network) ────────────────────────── + + +def test_get_feature_info_parses_response(monkeypatch: pytest.MonkeyPatch) -> None: + """get_feature_info вызывает _http_get_json с правильным URL + парсит features.""" + captured_urls: list[str] = [] + + def fake_http(url: str, **kwargs: Any) -> dict[str, Any]: + captured_urls.append(url) + return { + "type": "FeatureCollection", + "features": [ + { + "id": "tz-1", + "geometry": {"type": "Polygon", "coordinates": []}, + "properties": {"zone_code": "Ж-1"}, + } + ], + } + + monkeypatch.setattr("app.services.scrapers.nspd_client._http_get_json", fake_http) + client = NSPDClient() + feats = client.get_feature_info(875838, lon=60.605, lat=56.838, buffer_m=100) + assert len(feats) == 1 + assert feats[0].properties["zone_code"] == "Ж-1" + # Verify URL structure + assert len(captured_urls) == 1 + url = captured_urls[0] + assert "/api/aeggis/v4/875838/wms?" in url + assert "REQUEST=GetFeatureInfo" in url + assert "INFO_FORMAT=application%2Fjson" in url + assert "STYLES=" in url # required per NSPD WAF + assert "CRS=EPSG%3A3857" in url + # bbox should reflect 100m buffer (200m side) + assert "BBOX=" in url + + +def test_get_features_in_bbox_url_builds_correctly(monkeypatch: pytest.MonkeyPatch) -> None: + """get_features_in_bbox использует переданный bbox без преобразования.""" + captured_urls: list[str] = [] + + def fake_http(url: str, **kwargs: Any) -> dict[str, Any]: + captured_urls.append(url) + return {"features": []} + + monkeypatch.setattr("app.services.scrapers.nspd_client._http_get_json", fake_http) + client = NSPDClient() + bbox = (6700000.0, 7700000.0, 6800000.0, 7800000.0) # ЕКБ region in 3857 + feats = client.get_features_in_bbox(875838, bbox) + assert feats == [] + assert len(captured_urls) == 1 + url = captured_urls[0] + # bbox passed directly into BBOX param (URL-encoded comma = %2C) + assert "BBOX=6700000.0%2C7700000.0%2C6800000.0%2C7800000.0" in url + # Large width/height для bulk fetch + assert "WIDTH=4096" in url + assert "HEIGHT=4096" in url + # I/J pointer в центр виртуального тайла + assert "I=2048" in url + assert "J=2048" in url + + +def test_list_layers_walks_tree_response(monkeypatch: pytest.MonkeyPatch) -> None: + """list_layers вызывает /layers-theme-tree и распаковывает в NSPDLayer.""" + fake_tree = [ + { + "id": 1, + "title": "PKK тема", + "children": [ + { + "id": 875838, + "title": "Территориальные зоны", + "layerType": "wms", + "children": [], + }, + { + "id": 37577, + "title": "ЗОУИТ ОКН", + "layerType": "wms", + "children": [], + }, + ], + } + ] + monkeypatch.setattr( + "app.services.scrapers.nspd_client._http_get_json", + lambda url, **kw: fake_tree, + ) + client = NSPDClient() + layers = client.list_layers(theme_id=1) + layer_ids = [layer.layer_id for layer in layers] + # Root + 2 children = 3 leaves + assert 875838 in layer_ids + assert 37577 in layer_ids + # Sample layer metadata preserved + tz_layer = next(layer for layer in layers if layer.layer_id == 875838) + assert tz_layer.title == "Территориальные зоны" + assert tz_layer.layer_type == "wms" + + +def test_list_layers_handles_data_wrapper(monkeypatch: pytest.MonkeyPatch) -> None: + """Shape defense — NSPD иногда оборачивает в {"data": [...]}.""" + fake_wrapped = { + "data": [{"id": 36048, "title": "ЗУ", "children": []}], + "meta": {"count": 1}, + } + monkeypatch.setattr( + "app.services.scrapers.nspd_client._http_get_json", + lambda url, **kw: fake_wrapped, + ) + layers = NSPDClient().list_layers(theme_id=1) + assert len(layers) >= 1 + assert any(layer.layer_id == 36048 for layer in layers) + + +def test_list_layers_handles_garbage_response(monkeypatch: pytest.MonkeyPatch) -> None: + """Не dict/list response → empty list + warning log (no crash).""" + monkeypatch.setattr( + "app.services.scrapers.nspd_client._http_get_json", + lambda url, **kw: "garbage string", + ) + assert NSPDClient().list_layers(theme_id=1) == []