From 9ee0b070031b01ae8439ab1ac9e2a6d03dec2354 Mon Sep 17 00:00:00 2001 From: lekss361 <47113017+lekss361@users.noreply.github.com> Date: Tue, 12 May 2026 18:11:30 +0300 Subject: [PATCH] feat(scrapers): search_by_quarter orchestrator + QuarterDump (#94 pt.2/4) (#109) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sprint 1.1 item #1 из плана #94 part 2. Foundation для PKK harvest pipeline — 1 vacuum (search) + N layer fetches → comprehensive snapshot всех NSPD данных в пределах квартала. Базис для G1 #28 ПЗЗ, G3 #30 ЗОУИТ, P2 #46 neighbors, E1 #51 parcels backfill, #96 ЕГРН помещения. Backend (nspd_client.py): - New QuarterDump frozen dataclass (slots=True): quarter + per-layer feature lists (parcels/buildings/territorial_zones/red_lines/engineering + zouit dict + risks dict) + bbox_3857 + layers_fetched (immutable tuple) + fetched_at_utc + total_features property. - New NSPDClient.search_by_quarter(quarter_cad, include_zouit=True, include_risks=False): search → bbox → bulk fetch per layer phase. Cost 6/11/22 requests. - New _geojson_bbox_3857() module-level helper — recursive coord walker. - Class constants QUARTER_CORE_LAYERS / QUARTER_ZOUIT_LAYERS / QUARTER_RISK_LAYERS. Empty-quarter (NSPD не нашёл cad): quarter=None, bbox=None, all lists empty, zouit/risks dicts populated с пустыми lists (структурно стабильно), layers_fetched=('search',). Tests: +12 tests (31 total, no network). Code review (code-reviewer pre-push): MINOR, fixed 3 of 5: - datetime import → module-level - layers_fetched → tuple[str, ...] (immutable in frozen dataclass) - docstring clarified empty-quarter semantics Bonus: ruff UP038 isinstance tuple → union syntax. Part of #94. Sprint 1.1: 4 PRs total. Next: migration → Celery → integration. Co-authored-by: lekss361 --- backend/app/services/scrapers/nspd_client.py | 228 +++++++++++++ backend/tests/test_nspd_client.py | 335 +++++++++++++++++++ 2 files changed, 563 insertions(+) diff --git a/backend/app/services/scrapers/nspd_client.py b/backend/app/services/scrapers/nspd_client.py index 1dae05d4..da78fdf0 100644 --- a/backend/app/services/scrapers/nspd_client.py +++ b/backend/app/services/scrapers/nspd_client.py @@ -31,6 +31,7 @@ WMS endpoints (per #94 issue body, TIER 1-6 каталог слоёв): from __future__ import annotations +import datetime as _dt import json import logging import math @@ -170,6 +171,53 @@ class NSPDLayer: metadata: dict[str, Any] +@dataclass(frozen=True, slots=True) +class QuarterDump: + """Comprehensive snapshot of NSPD data в пределах одного quarter. + + Foundation для PKK harvest pipeline (#94 follow-up): + - `quarter` — собственно квартал (geometry в EPSG:3857) + - per-layer feature lists в пределах квартала (bbox-filtered) + + Один dump = достаточно данных чтобы analyze_parcel мог отдать ВСЕ Gate- + факторы (G1 ПЗЗ, G3 ЗОУИТ), neighbors, инженерку — без новых NSPD-вызовов + в request-цикле. Беём дешевле для пользователя (1 HTTP вместо N). + + Layers распределены по 3 phases: + - core: parcels + buildings + territorial_zones + red_lines + engineering + - zouit: 5 ЗОУИТ layers (G3) + - risks: 11 risk-zone layers (TIER 3) + Default = только core, чтобы не сжигать rate-limit на 17 запросов. + """ + + quarter_cad: str + quarter: NSPDFeature | None # сам polygon квартала (может быть None если NSPD пуст) + parcels: list[NSPDFeature] + buildings: list[NSPDFeature] + territorial_zones: list[NSPDFeature] # ПЗЗ зоны покрывающие/пересекающие + red_lines: list[NSPDFeature] + engineering_structures: list[NSPDFeature] + zouit: dict[str, list[NSPDFeature]] # {"okn": [...], "engineering": [...], ...} + risks: dict[str, list[NSPDFeature]] # {"flooding": [...], "landslide": [...], ...} + # tuple, не list — frozen dataclass + immutable contents (audit/debug snapshot) + layers_fetched: tuple[str, ...] + bbox_3857: tuple[float, float, float, float] | None # bbox квартала + fetched_at_utc: str # ISO timestamp когда сделали запрос (для freshness check) + + @property + def total_features(self) -> int: + """Сумма всех features во всех layers — для smoke check.""" + return ( + len(self.parcels) + + len(self.buildings) + + len(self.territorial_zones) + + len(self.red_lines) + + len(self.engineering_structures) + + sum(len(v) for v in self.zouit.values()) + + sum(len(v) for v in self.risks.values()) + ) + + # ── HTTP helper ────────────────────────────────────────────────────────────── @@ -410,6 +458,185 @@ class NSPDClient: ) return layers + # ── 5. search_by_quarter (PKK harvest orchestrator) ───────────────────── + + # Layer groupings для search_by_quarter. Назван "phase" т.к. в Celery + # task может включаться выборочно (rate-limit budget). + QUARTER_CORE_LAYERS: dict[str, str] = { # noqa: RUF012 — class-level constant + "parcels": "parcels", + "buildings": "buildings", + "territorial_zones": "territorial_zones", + "red_lines": "red_lines", + "engineering_structures": "engineering_structures", + } + QUARTER_ZOUIT_LAYERS: dict[str, str] = { # noqa: RUF012 + "okn": "zouit_okn", + "engineering": "zouit_engineering", + "natural": "zouit_natural", + "protected": "zouit_protected", + "other": "zouit_other", + } + QUARTER_RISK_LAYERS: dict[str, str] = { # noqa: RUF012 + "flooding_underground": "risk_flooding_underground", + "flooding": "risk_flooding", + "swampification": "risk_swampification", + "landslide": "risk_landslide", + "abrasion": "risk_abrasion", + "erosion_water": "risk_erosion_water", + "erosion_linear": "risk_erosion_linear", + "erosion_wind": "risk_erosion_wind", + "desertification": "risk_desertification", + "clutter": "risk_clutter", + "burns": "risk_burns", + } + + def search_by_quarter( + self, + quarter_cad: str, + *, + include_zouit: bool = True, + include_risks: bool = False, + ) -> QuarterDump: + """Harvest всех NSPD-данных для квартала: 1 vacuum, N layers. + + Шаги: + 1. `search_by_cad(quarter_cad, thematic_id=2)` — получить полигон квартала + 2. Compute bbox в EPSG:3857 из quarter geometry (или None если NSPD пуст) + 3. Для каждого core layer → `get_features_in_bbox(layer_id, bbox)` + 4. Если include_zouit — то же для 5 ЗОУИТ layers + 5. Если include_risks — то же для 11 risk layers + + Стоимость HTTP: + - core only: 1 (search) + 5 (core layers) = 6 запросов + - +zouit: +5 = 11 запросов + - +risks: +11 = 22 запроса + При rate_ms=600 один dump = ~3.6с (core) / ~6.6с (+zouit) / ~13с (всё). + + Args: + quarter_cad: 3-сегментный cad-номер квартала, e.g. '66:41:0204016'. + include_zouit: Включать TIER 2 ЗОУИТ layers (G3). Default True. + include_risks: Включать TIER 3 risk zones. Default False (rate-limit + budget; для отдельного D-N risk score можно включить). + + Returns: + QuarterDump с per-layer feature lists. Если NSPD пуст / quarter + не найден — `quarter=None`, `bbox_3857=None`, все feature lists + пустые (no bulk-fetch без bounds — нет смысла). При этом dict- + поля `zouit` / `risks` всё равно populated с пустыми lists для + каждого включённого short_name (структура контракта стабильна). + `layers_fetched` в этом случае содержит только `('search',)`. + + Raises: + NspdLiteWafError при 403/429 на любом из layer запросов — caller + должен делать backoff. Partial-success НЕ возвращается; вся + операция атомарна (failure → exception). + + Закрывает: foundation для G1 #28 ПЗЗ, G3 #30 ЗОУИТ, P2 #46 neighbors, + E1 #51 parcels backfill, #96 ЕГРН помещения. + """ + # 1. Quarter geometry через REST search + quarter_search = self.search_by_cad(quarter_cad, thematic_id=2) + quarter_feat: NSPDFeature | None = quarter_search.first + + # 2. Compute bbox в 3857. Если NSPD ничего не нашёл — bbox=None, + # все layer-запросы возвращаем как empty (нет смысла bulk-fetch'ить + # без bounds). + bbox: tuple[float, float, float, float] | None = None + if quarter_feat and quarter_feat.geometry: + bbox = _geojson_bbox_3857(quarter_feat.geometry) + + layers_fetched: list[str] = ["search"] + + def _fetch_layer(name_in_dump: str, layer_key: str) -> list[NSPDFeature]: + """Helper: безопасно получить features для одного layer.""" + if bbox is None: + return [] + layer_id = LAYERS.get(layer_key) + if layer_id is None: + logger.warning("search_by_quarter: unknown layer key %s", layer_key) + return [] + layers_fetched.append(name_in_dump) + return self.get_features_in_bbox(layer_id, bbox) + + # 3. Core layers + parcels = _fetch_layer("parcels", "parcels") + buildings = _fetch_layer("buildings", "buildings") + territorial_zones = _fetch_layer("territorial_zones", "territorial_zones") + red_lines = _fetch_layer("red_lines", "red_lines") + engineering_structures = _fetch_layer("engineering_structures", "engineering_structures") + + # 4. ЗОУИТ (G3) + zouit: dict[str, list[NSPDFeature]] = {} + if include_zouit: + for short_name, layer_key in self.QUARTER_ZOUIT_LAYERS.items(): + zouit[short_name] = _fetch_layer(f"zouit_{short_name}", layer_key) + + # 5. Risks (TIER 3) + risks: dict[str, list[NSPDFeature]] = {} + if include_risks: + for short_name, layer_key in self.QUARTER_RISK_LAYERS.items(): + risks[short_name] = _fetch_layer(f"risk_{short_name}", layer_key) + + return QuarterDump( + quarter_cad=quarter_cad, + quarter=quarter_feat, + parcels=parcels, + buildings=buildings, + territorial_zones=territorial_zones, + red_lines=red_lines, + engineering_structures=engineering_structures, + zouit=zouit, + risks=risks, + layers_fetched=tuple(layers_fetched), + bbox_3857=bbox, + fetched_at_utc=_dt.datetime.now(_dt.UTC).isoformat(), + ) + + +# ── Geometry helpers (module-level — для unit-test'ов) ────────────────────── + + +def _geojson_bbox_3857( + geometry: dict[str, Any], +) -> tuple[float, float, float, float] | None: + """Compute bbox в EPSG:3857 из GeoJSON geometry. + + NSPD search /v2 возвращает coordinates в EPSG:3857 (Web Mercator metres) — + напрямую compute min/max по координатам. Если geometry уже в WGS84 + (нестандартный сценарий) — caller должен трансформировать заранее. + + Поддерживает Polygon/MultiPolygon/LineString/Point. + Возвращает None для пустой/некорректной geometry. + """ + coords = geometry.get("coordinates") + geom_type = geometry.get("type") + if not coords or not geom_type: + return None + + def _walk(node: Any) -> list[tuple[float, float]]: + """Рекурсивный extract координат до получения list of (x, y) tuples.""" + if isinstance(node, int | float): + return [] + if ( + isinstance(node, list) + and len(node) >= 2 + and all(isinstance(v, int | float) for v in node[:2]) + ): + return [(float(node[0]), float(node[1]))] + if isinstance(node, list): + out: list[tuple[float, float]] = [] + for sub in node: + out.extend(_walk(sub)) + return out + return [] + + pts = _walk(coords) + if not pts: + return None + xs = [p[0] for p in pts] + ys = [p[1] for p in pts] + return (min(xs), min(ys), max(xs), max(ys)) + def _walk_layer_tree(node: Any) -> list[dict[str, Any]]: """Рекурсивный walker для NSPD layers-theme-tree. Yields leaf nodes.""" @@ -444,6 +671,7 @@ __all__ = [ "NSPDSearchResult", "NspdLiteError", "NspdLiteWafError", + "QuarterDump", "bbox_around_point_m", "lonlat_to_3857", ] diff --git a/backend/tests/test_nspd_client.py b/backend/tests/test_nspd_client.py index 0a862234..b7b7479b 100644 --- a/backend/tests/test_nspd_client.py +++ b/backend/tests/test_nspd_client.py @@ -18,6 +18,8 @@ from app.services.scrapers.nspd_client import ( NSPDFeature, NSPDLayer, NSPDSearchResult, + QuarterDump, + _geojson_bbox_3857, _walk_layer_tree, bbox_around_point_m, lonlat_to_3857, @@ -319,3 +321,336 @@ def test_list_layers_handles_garbage_response(monkeypatch: pytest.MonkeyPatch) - lambda url, **kw: "garbage string", ) assert NSPDClient().list_layers(theme_id=1) == [] + + +# ── _geojson_bbox_3857 tests ────────────────────────────────────────────────── + + +def test_geojson_bbox_3857_polygon() -> None: + """Простой Polygon → bbox охватывает все vertices.""" + geom = { + "type": "Polygon", + "coordinates": [[[0, 0], [100, 0], [100, 100], [0, 100], [0, 0]]], + } + result = _geojson_bbox_3857(geom) + assert result == (0.0, 0.0, 100.0, 100.0) + + +def test_geojson_bbox_3857_multipolygon() -> None: + """MultiPolygon с двумя кусками → bbox охватывает оба.""" + geom = { + "type": "MultiPolygon", + "coordinates": [ + [[[0, 0], [10, 0], [10, 10], [0, 10], [0, 0]]], + [[[20, 30], [50, 30], [50, 60], [20, 60], [20, 30]]], + ], + } + result = _geojson_bbox_3857(geom) + assert result is not None + xmin, ymin, xmax, ymax = result + assert xmin == 0.0 + assert ymin == 0.0 + assert xmax == 50.0 + assert ymax == 60.0 + + +def test_geojson_bbox_3857_linestring() -> None: + """LineString из 2 точек → bbox прямоугольник вокруг них.""" + geom = { + "type": "LineString", + "coordinates": [[10, 20], [30, 40]], + } + result = _geojson_bbox_3857(geom) + assert result == (10.0, 20.0, 30.0, 40.0) + + +def test_geojson_bbox_3857_point() -> None: + """Point → bbox с шириной и высотой 0 (один пиксель).""" + geom = { + "type": "Point", + "coordinates": [60.0, 56.0], + } + result = _geojson_bbox_3857(geom) + assert result == (60.0, 56.0, 60.0, 56.0) + + +def test_geojson_bbox_3857_empty() -> None: + """Polygon с пустым coordinates → None.""" + geom = {"type": "Polygon", "coordinates": []} + assert _geojson_bbox_3857(geom) is None + + +def test_geojson_bbox_3857_no_type() -> None: + """Нет поля type → None (не можем определить геометрию).""" + geom: dict[str, Any] = {"coordinates": [[0, 0]]} + assert _geojson_bbox_3857(geom) is None + + +# ── QuarterDump tests ───────────────────────────────────────────────────────── + + +def _make_feat(fid: str = "x") -> NSPDFeature: + """Вспомогательная фабрика NSPDFeature для тестов.""" + return NSPDFeature.from_raw({"id": fid, "geometry": None, "properties": {}}) + + +def test_quarter_dump_total_features() -> None: + """total_features корректно суммирует features по всем layers.""" + dump = QuarterDump( + quarter_cad="66:41:0204016", + quarter=_make_feat("q"), + parcels=[_make_feat("p1"), _make_feat("p2"), _make_feat("p3")], # 3 + buildings=[_make_feat("b1")], # 1 + territorial_zones=[_make_feat("tz1"), _make_feat("tz2")], # 2 + red_lines=[], # 0 + engineering_structures=[_make_feat("e1")], # 1 + zouit={ + "okn": [_make_feat("ok1"), _make_feat("ok2")], # 2 + "natural": [_make_feat("nat1")], # 1 + }, + risks={ + "flooding": [_make_feat("fl1"), _make_feat("fl2"), _make_feat("fl3")], # 3 + "landslide": [_make_feat("ls1")], # 1 + }, + layers_fetched=("search", "parcels", "buildings"), + bbox_3857=(6700000.0, 7700000.0, 6800000.0, 7800000.0), + fetched_at_utc="2026-05-12T00:00:00+00:00", + ) + # 3+1+2+0+1 (core) + 2+1 (zouit) + 3+1 (risks) = 14 + assert dump.total_features == 14 + + +def test_quarter_dump_frozen() -> None: + """QuarterDump frozen=True: попытка assign → AttributeError.""" + dump = QuarterDump( + quarter_cad="66:41:0204016", + quarter=None, + parcels=[], + buildings=[], + territorial_zones=[], + red_lines=[], + engineering_structures=[], + zouit={}, + risks={}, + layers_fetched=("search",), + bbox_3857=None, + fetched_at_utc="2026-05-12T00:00:00+00:00", + ) + with pytest.raises(AttributeError): + dump.parcels = [] # type: ignore[misc] + + +# ── search_by_quarter mock tests ────────────────────────────────────────────── + +# Квартал в районе ЕКБ (EPSG:3857 метры) +_QUARTER_COORDS = [ + [6700000.0, 7700000.0], + [6800000.0, 7700000.0], + [6800000.0, 7800000.0], + [6700000.0, 7800000.0], + [6700000.0, 7700000.0], +] +_QUARTER_BBOX = (6700000.0, 7700000.0, 6800000.0, 7800000.0) + +_FAKE_QUARTER_SEARCH = { + "data": { + "type": "FeatureCollection", + "features": [ + { + "id": "q-1", + "geometry": {"type": "Polygon", "coordinates": [_QUARTER_COORDS]}, + "properties": {"cad_num": "66:41:0204016"}, + } + ], + } +} + +_LAYER_FEATURE_COUNTS: dict[str, int] = { + "parcels": 4, + "buildings": 2, + "territorial_zones": 1, + "red_lines": 0, + "engineering_structures": 3, +} + + +def _make_fake_http( + layer_feature_counts: dict[str, int] | None = None, +) -> Any: + """Возвращает fake _http_get_json который генерирует N features по layer ID. + + layer_feature_counts: {layer_name: count} — если None, возвращает 1 feature. + """ + counts = layer_feature_counts or {} + # Строим reverse-map layer_id → count для подстановки в URL + from app.services.scrapers.nspd_client import LAYERS as _LAYERS + + id_to_name: dict[int, str] = {v: k for k, v in _LAYERS.items()} + + def fake_http(url: str, **kwargs: Any) -> dict[str, Any]: + # Извлекаем layer_id из URL вида /api/aeggis/v4/{id}/wms?... + parts = url.split("/") + layer_id_str = parts[6] if len(parts) > 6 else "0" + try: + layer_id = int(layer_id_str.split("?")[0]) + except ValueError: + layer_id = 0 + layer_name = id_to_name.get(layer_id, "unknown") + n = counts.get(layer_name, 1) + return { + "type": "FeatureCollection", + "features": [ + {"id": f"{layer_name}-{i}", "geometry": None, "properties": {}} for i in range(n) + ], + } + + return fake_http + + +def test_search_by_quarter_core_only(monkeypatch: pytest.MonkeyPatch) -> None: + """core_only (include_zouit=False, include_risks=False): 1 search + 5 bulk.""" + search_calls: list[str] = [] + + def fake_fetch(query: str, **kwargs: Any) -> dict[str, Any]: + search_calls.append(query) + return _FAKE_QUARTER_SEARCH + + monkeypatch.setattr("app.services.scrapers.nspd_client.fetch_geoportal", fake_fetch) + monkeypatch.setattr( + "app.services.scrapers.nspd_client._http_get_json", + _make_fake_http(_LAYER_FEATURE_COUNTS), + ) + + result = NSPDClient().search_by_quarter( + "66:41:0204016", include_zouit=False, include_risks=False + ) + + # Quarter populated + assert result.quarter is not None + assert result.quarter.feature_id == "q-1" + + # Core layer lengths match mock counts + assert len(result.parcels) == 4 + assert len(result.buildings) == 2 + assert len(result.territorial_zones) == 1 + assert len(result.red_lines) == 0 + assert len(result.engineering_structures) == 3 + + # ЗОУИТ / risks пусты + assert result.zouit == {} + assert result.risks == {} + + # layers_fetched: search + 5 core (red_lines тоже: _fetch_layer вызывается, + # но возвращает пустой список — имя всё равно добавляется). tuple, not list + # — frozen-dataclass immutability invariant. + assert result.layers_fetched == ( + "search", + "parcels", + "buildings", + "territorial_zones", + "red_lines", + "engineering_structures", + ) + + # bbox вычислен из четырёхугольного polygon + assert result.bbox_3857 == _QUARTER_BBOX + + +def test_search_by_quarter_with_zouit(monkeypatch: pytest.MonkeyPatch) -> None: + """include_zouit=True → zouit dict содержит 5 ключей.""" + monkeypatch.setattr( + "app.services.scrapers.nspd_client.fetch_geoportal", + lambda *a, **kw: _FAKE_QUARTER_SEARCH, + ) + monkeypatch.setattr( + "app.services.scrapers.nspd_client._http_get_json", + _make_fake_http(), + ) + + result = NSPDClient().search_by_quarter( + "66:41:0204016", include_zouit=True, include_risks=False + ) + + assert set(result.zouit.keys()) == {"okn", "engineering", "natural", "protected", "other"} + assert result.risks == {} + + # layers_fetched содержит search + 5 core + 5 zouit = 11 записей + assert len(result.layers_fetched) == 11 + + +def test_search_by_quarter_empty_quarter(monkeypatch: pytest.MonkeyPatch) -> None: + """Пустой FeatureCollection → quarter=None, все layer feature lists пустые. + + При bbox=None `_fetch_layer` возвращает [] без HTTP вызовов. ЗОУИТ-словарь + всё равно создаётся (с пустыми списками) — ключи присутствуют, features нет. + """ + http_calls: list[str] = [] + + def fake_http(url: str, **kwargs: Any) -> dict[str, Any]: + http_calls.append(url) + return {"type": "FeatureCollection", "features": []} + + monkeypatch.setattr( + "app.services.scrapers.nspd_client.fetch_geoportal", + lambda *a, **kw: {"data": {"type": "FeatureCollection", "features": []}}, + ) + monkeypatch.setattr("app.services.scrapers.nspd_client._http_get_json", fake_http) + + result = NSPDClient().search_by_quarter("66:41:9999999", include_zouit=True) + + assert result.quarter is None + assert result.parcels == [] + assert result.buildings == [] + assert result.territorial_zones == [] + assert result.red_lines == [] + assert result.engineering_structures == [] + + # include_zouit=True: ключи созданы, но все списки пустые (bbox=None → _fetch_layer → []) + assert set(result.zouit.keys()) == {"okn", "engineering", "natural", "protected", "other"} + assert all(v == [] for v in result.zouit.values()) + + assert result.risks == {} + assert result.bbox_3857 is None + + # Никаких bulk HTTP запросов — bbox=None ⇒ ранний выход из _fetch_layer + assert http_calls == [] + # layers_fetched содержит только "search" (tuple, no bulk fetches happened) + assert result.layers_fetched == ("search",) + + +def test_search_by_quarter_layers_fetched_with_risks(monkeypatch: pytest.MonkeyPatch) -> None: + """include_risks=True (без zouit) → layers_fetched включает 11 risk layer имён.""" + monkeypatch.setattr( + "app.services.scrapers.nspd_client.fetch_geoportal", + lambda *a, **kw: _FAKE_QUARTER_SEARCH, + ) + monkeypatch.setattr( + "app.services.scrapers.nspd_client._http_get_json", + _make_fake_http(), + ) + + result = NSPDClient().search_by_quarter( + "66:41:0204016", include_zouit=False, include_risks=True + ) + + assert result.zouit == {} + assert set(result.risks.keys()) == { + "flooding_underground", + "flooding", + "swampification", + "landslide", + "abrasion", + "erosion_water", + "erosion_linear", + "erosion_wind", + "desertification", + "clutter", + "burns", + } + + # layers_fetched: search + 5 core + 11 risks = 17 + assert len(result.layers_fetched) == 17 + # Убедимся что risk имена действительно присутствуют в layers_fetched + assert "risk_flooding" in result.layers_fetched + assert "risk_landslide" in result.layers_fetched + assert "risk_burns" in result.layers_fetched