fix(scrapers): address PR #98 auto-review minor feedback
Backend (nspd_client.py):
1. (line 100+) bbox_around_point_m signature split на multi-line.
2. (lazy imports) import json, time → module-level.
3. (duplicate SSL ctx) Reuse _SSL_CTX из nspd_lite через explicit import.
4. (shape defense) list_layers чекает {"data": [...]} wrapper + non-dict/list garbage → warning + empty.
5. (cad_num docstring) search_by_cad документирует формат + ссылку на validate_cad_format.
Tests: +5 mock тестов (19/19 passed):
- get_feature_info URL builder + parsing
- get_features_in_bbox bbox propagation
- list_layers_walks_tree_response
- list_layers_handles_data_wrapper
- list_layers_handles_garbage_response
Backend (parcels.py): docstring '~25с' → '~15с'.
Per auto-review on be14388.
This commit is contained in:
lekss361
parent
be143880d0
commit
6ce73221a9
3 changed files with 156 additions and 9 deletions
|
|
@ -1020,7 +1020,7 @@ def analyze_parcel(
|
|||
|
||||
Issue #93 — Graceful fallback при отсутствии geometry:
|
||||
- Не возвращаем 404 сразу. Вместо: enqueue NSPD on-demand fetch, ждём
|
||||
inline до _INLINE_FETCH_WAIT_S (~25с). Если за это время геометрия
|
||||
inline до _INLINE_FETCH_WAIT_S (~15с). Если за это время геометрия
|
||||
появилась в БД — продолжаем analyze (fast path).
|
||||
- Иначе → 202 Accepted + {status, job_id, eta_seconds} для polling
|
||||
через GET /fetch-status.
|
||||
|
|
|
|||
|
|
@ -31,9 +31,10 @@ WMS endpoints (per #94 issue body, TIER 1-6 каталог слоёв):
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import ssl
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
|
@ -41,6 +42,7 @@ from dataclasses import dataclass
|
|||
from typing import Any
|
||||
|
||||
from app.services.scrapers.nspd_lite import (
|
||||
_SSL_CTX,
|
||||
HEADERS,
|
||||
NspdLiteError,
|
||||
NspdLiteWafError,
|
||||
|
|
@ -109,8 +111,8 @@ LAYERS: dict[str, int] = {
|
|||
# Default rate limit (мс между запросами) — баланс между скоростью и WAF
|
||||
DEFAULT_RATE_MS = 600
|
||||
|
||||
# SSL context — reuse from nspd_lite pattern (WAF-compatible)
|
||||
_SSL_CTX = ssl._create_unverified_context()
|
||||
# SSL context — reuse `_SSL_CTX` из nspd_lite (one source of truth для WAF-
|
||||
# compatible TLS settings; нет дублирования если что-то поменяется).
|
||||
|
||||
|
||||
# ── Typed responses ──────────────────────────────────────────────────────────
|
||||
|
|
@ -178,9 +180,6 @@ def _http_get_json(url: str, *, timeout: int = 15, rate_ms: int = 0) -> Any:
|
|||
layers-tree). Reuse HEADERS + SSL context из nspd_lite — общая защита от
|
||||
WAF блокировки по TLS fingerprint.
|
||||
"""
|
||||
import json
|
||||
import time
|
||||
|
||||
if rate_ms > 0:
|
||||
time.sleep(rate_ms / 1000.0)
|
||||
req = urllib.request.Request(url, headers=HEADERS)
|
||||
|
|
@ -212,7 +211,11 @@ def lonlat_to_3857(lon: float, lat: float) -> tuple[float, float]:
|
|||
return (x, y)
|
||||
|
||||
|
||||
def bbox_around_point_m(lon: float, lat: float, buffer_m: int) -> tuple[float, float, float, float]:
|
||||
def bbox_around_point_m(
|
||||
lon: float,
|
||||
lat: float,
|
||||
buffer_m: int,
|
||||
) -> tuple[float, float, float, float]:
|
||||
"""Микро-bbox вокруг точки в EPSG:3857 — для WMS GetFeatureInfo.
|
||||
|
||||
Returns (xmin, ymin, xmax, ymax). buffer_m по m в каждую сторону.
|
||||
|
|
@ -241,7 +244,15 @@ class NSPDClient:
|
|||
"""REST поиск /v2/search/geoportal — 1 запрос = geometry + ВРИ + кост.
|
||||
|
||||
Args:
|
||||
cad_num: '66:41:0204016' (квартал) или '66:41:0204016:10' (участок).
|
||||
cad_num: канонический формат — `NN:NN:NNNNNN[:NN[:NN]]`, цифры через `:`.
|
||||
Примеры:
|
||||
- 3-сегментный квартал: `'66:41:0204016'`
|
||||
- 4-сегментный участок: `'66:41:0204016:10'`
|
||||
- 5-сегментное здание: `'66:41:0204016:10:1'`
|
||||
Спецсимволы / пробелы корректно URL-encode'нутся (нет SQL/URL
|
||||
injection), но НСПД молча вернёт empty result. Caller обязан
|
||||
сам валидировать формат если нужна defensive обработка (см.
|
||||
`app.services.site_finder.cadastre_fetch.validate_cad_format`).
|
||||
thematic_id: 1 ЗУ / 2 квартал / 5 здание / 7 территориальная зона.
|
||||
Default 1 — соответствует use-case G2 ВРИ.
|
||||
|
||||
|
|
@ -373,6 +384,18 @@ class NSPDClient:
|
|||
"""
|
||||
url = f"{NSPD_LAYERS_TREE}?themeId={theme_id}"
|
||||
data = _http_get_json(url, timeout=self.timeout, rate_ms=self.rate_ms)
|
||||
# Shape defense: NSPD иногда оборачивает в {"data": [...]} — паттерн уже
|
||||
# ловили в Bug_Nspd_Geo_Str_Object_No_Get_Fixed. Без проверки walker
|
||||
# молча вернул бы [], а caller увидел бы пустой catalog.
|
||||
if isinstance(data, dict) and "data" in data and not data.get("children"):
|
||||
data = data.get("data") or []
|
||||
if not isinstance(data, list | dict):
|
||||
logger.warning(
|
||||
"list_layers: unexpected NSPD response type %s for theme_id=%s",
|
||||
type(data).__name__,
|
||||
theme_id,
|
||||
)
|
||||
return []
|
||||
layers: list[NSPDLayer] = []
|
||||
# Tree response: nested {"children": [...], "id": N, "title": "..."}
|
||||
# Используем walker по subtree чтобы вытащить все leaf layers
|
||||
|
|
|
|||
|
|
@ -195,3 +195,127 @@ def test_nspd_layer_dataclass() -> None:
|
|||
# Frozen
|
||||
with pytest.raises(AttributeError):
|
||||
layer.title = "x" # type: ignore[misc]
|
||||
|
||||
|
||||
# ── Mock tests for WMS/layers methods (no network) ──────────────────────────
|
||||
|
||||
|
||||
def test_get_feature_info_parses_response(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""get_feature_info вызывает _http_get_json с правильным URL + парсит features."""
|
||||
captured_urls: list[str] = []
|
||||
|
||||
def fake_http(url: str, **kwargs: Any) -> dict[str, Any]:
|
||||
captured_urls.append(url)
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"id": "tz-1",
|
||||
"geometry": {"type": "Polygon", "coordinates": []},
|
||||
"properties": {"zone_code": "Ж-1"},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
monkeypatch.setattr("app.services.scrapers.nspd_client._http_get_json", fake_http)
|
||||
client = NSPDClient()
|
||||
feats = client.get_feature_info(875838, lon=60.605, lat=56.838, buffer_m=100)
|
||||
assert len(feats) == 1
|
||||
assert feats[0].properties["zone_code"] == "Ж-1"
|
||||
# Verify URL structure
|
||||
assert len(captured_urls) == 1
|
||||
url = captured_urls[0]
|
||||
assert "/api/aeggis/v4/875838/wms?" in url
|
||||
assert "REQUEST=GetFeatureInfo" in url
|
||||
assert "INFO_FORMAT=application%2Fjson" in url
|
||||
assert "STYLES=" in url # required per NSPD WAF
|
||||
assert "CRS=EPSG%3A3857" in url
|
||||
# bbox should reflect 100m buffer (200m side)
|
||||
assert "BBOX=" in url
|
||||
|
||||
|
||||
def test_get_features_in_bbox_url_builds_correctly(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""get_features_in_bbox использует переданный bbox без преобразования."""
|
||||
captured_urls: list[str] = []
|
||||
|
||||
def fake_http(url: str, **kwargs: Any) -> dict[str, Any]:
|
||||
captured_urls.append(url)
|
||||
return {"features": []}
|
||||
|
||||
monkeypatch.setattr("app.services.scrapers.nspd_client._http_get_json", fake_http)
|
||||
client = NSPDClient()
|
||||
bbox = (6700000.0, 7700000.0, 6800000.0, 7800000.0) # ЕКБ region in 3857
|
||||
feats = client.get_features_in_bbox(875838, bbox)
|
||||
assert feats == []
|
||||
assert len(captured_urls) == 1
|
||||
url = captured_urls[0]
|
||||
# bbox passed directly into BBOX param (URL-encoded comma = %2C)
|
||||
assert "BBOX=6700000.0%2C7700000.0%2C6800000.0%2C7800000.0" in url
|
||||
# Large width/height для bulk fetch
|
||||
assert "WIDTH=4096" in url
|
||||
assert "HEIGHT=4096" in url
|
||||
# I/J pointer в центр виртуального тайла
|
||||
assert "I=2048" in url
|
||||
assert "J=2048" in url
|
||||
|
||||
|
||||
def test_list_layers_walks_tree_response(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""list_layers вызывает /layers-theme-tree и распаковывает в NSPDLayer."""
|
||||
fake_tree = [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "PKK тема",
|
||||
"children": [
|
||||
{
|
||||
"id": 875838,
|
||||
"title": "Территориальные зоны",
|
||||
"layerType": "wms",
|
||||
"children": [],
|
||||
},
|
||||
{
|
||||
"id": 37577,
|
||||
"title": "ЗОУИТ ОКН",
|
||||
"layerType": "wms",
|
||||
"children": [],
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.nspd_client._http_get_json",
|
||||
lambda url, **kw: fake_tree,
|
||||
)
|
||||
client = NSPDClient()
|
||||
layers = client.list_layers(theme_id=1)
|
||||
layer_ids = [layer.layer_id for layer in layers]
|
||||
# Root + 2 children = 3 leaves
|
||||
assert 875838 in layer_ids
|
||||
assert 37577 in layer_ids
|
||||
# Sample layer metadata preserved
|
||||
tz_layer = next(layer for layer in layers if layer.layer_id == 875838)
|
||||
assert tz_layer.title == "Территориальные зоны"
|
||||
assert tz_layer.layer_type == "wms"
|
||||
|
||||
|
||||
def test_list_layers_handles_data_wrapper(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Shape defense — NSPD иногда оборачивает в {"data": [...]}."""
|
||||
fake_wrapped = {
|
||||
"data": [{"id": 36048, "title": "ЗУ", "children": []}],
|
||||
"meta": {"count": 1},
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.nspd_client._http_get_json",
|
||||
lambda url, **kw: fake_wrapped,
|
||||
)
|
||||
layers = NSPDClient().list_layers(theme_id=1)
|
||||
assert len(layers) >= 1
|
||||
assert any(layer.layer_id == 36048 for layer in layers)
|
||||
|
||||
|
||||
def test_list_layers_handles_garbage_response(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Не dict/list response → empty list + warning log (no crash)."""
|
||||
monkeypatch.setattr(
|
||||
"app.services.scrapers.nspd_client._http_get_json",
|
||||
lambda url, **kw: "garbage string",
|
||||
)
|
||||
assert NSPDClient().list_layers(theme_id=1) == []
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue