From fdced946266977aa410efc57cdd2807029c29b58 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 17 May 2026 09:46:42 +0300 Subject: [PATCH] fix(#231): switch search_by_quarter area layers to grid-walk dispatch (Sub-PR B) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add _GRID_WALK_LAYERS frozenset (territorial_zones, red_lines, engineering_structures, zouit_*, risk_*) - _fetch_layer dispatch: area layers → get_features_in_bbox_grid(grid_n=7, step_m=50) [foundation из PR #247], EGRN layers (parcels, buildings) → legacy get_features_in_bbox - Enrich engineering_structures features с classified_kind через classify_engineering_kind (PR #247 classifier) - Per-layer logger.info heartbeat (method + count) для прогресса - Bump harvest_quarter soft_time_limit 120 → 600s (grid-walk: 11 layers × 49 cells × ~70ms ≈ 40s + retries margin) - TestFetchLayerDispatch (3 tests): 34 passed, 1 skipped Pre-push deep-code-reviewer: APPROVE (1 medium nit — opportunity_oopt не в grid-walk, защищено через include_opportunity=False default; follow-up issue рекомендован). --- backend/app/services/scrapers/nspd_client.py | 68 ++++++++- backend/app/workers/tasks/nspd_sync.py | 2 +- backend/tests/scrapers/test_nspd_grid_walk.py | 135 +++++++++++++++++- 3 files changed, 201 insertions(+), 4 deletions(-) diff --git a/backend/app/services/scrapers/nspd_client.py b/backend/app/services/scrapers/nspd_client.py index 639ba3bb..40d1cbf8 100644 --- a/backend/app/services/scrapers/nspd_client.py +++ b/backend/app/services/scrapers/nspd_client.py @@ -43,6 +43,7 @@ import urllib.request from dataclasses import dataclass from typing import Any +from app.services.scrapers.nspd_denorm import classify_engineering_kind from app.services.scrapers.nspd_lite import ( _SSL_CTX, HEADERS, @@ -110,6 +111,37 @@ LAYERS: dict[str, int] = { } +# Layers where grid-walk (get_features_in_bbox_grid) must be used instead of +# the legacy single-pixel WMS probe (get_features_in_bbox). These are area/ +# linear layers (territorial zones, red lines, engineering structures, ЗОУИТ, +# risks) that span large areas and are under-returned by a single GetFeatureInfo +# call. Point/polygon EGRN layers (parcels, buildings) stay on legacy for now. +_GRID_WALK_LAYERS: frozenset[str] = frozenset( + { + "territorial_zones", + "red_lines", + "engineering_structures", + # ЗОУИТ (TIER 2) + "zouit_okn", + "zouit_engineering", + "zouit_natural", + "zouit_protected", + "zouit_other", + # Risks (TIER 3) + "risk_flooding_underground", + "risk_flooding", + "risk_swampification", + "risk_landslide", + "risk_abrasion", + "risk_erosion_water", + "risk_erosion_linear", + "risk_erosion_wind", + "risk_desertification", + "risk_clutter", + "risk_burns", + } +) + # Default rate limit (мс между запросами) — баланс между скоростью и WAF DEFAULT_RATE_MS = 600 @@ -709,7 +741,17 @@ class NSPDClient: layers_fetched: list[str] = ["search"] def _fetch_layer(name_in_dump: str, layer_key: str) -> list[NSPDFeature]: - """Helper: безопасно получить features для одного layer.""" + """Helper: безопасно получить features для одного layer. + + Dispatch: + - area/linear layers (_GRID_WALK_LAYERS) → grid-walk + (get_features_in_bbox_grid, grid_n=7, step_m=50) + - point/polygon EGRN layers (parcels, buildings, ons, enk) + → legacy single-pixel WMS (get_features_in_bbox) + + Engineering structures features дополнительно обогащаются + properties["classified_kind"] через classify_engineering_kind. + """ if bbox is None: return [] layer_id = LAYERS.get(layer_key) @@ -717,7 +759,29 @@ class NSPDClient: logger.warning("search_by_quarter: unknown layer key %s", layer_key) return [] layers_fetched.append(name_in_dump) - return self.get_features_in_bbox(layer_id, bbox) + if layer_key in _GRID_WALK_LAYERS: + features = self.get_features_in_bbox_grid(layer_id, bbox, grid_n=7, step_m=50.0) + # Обогатить engineering_structures classified_kind + if layer_key == "engineering_structures": + for feat in features: + feat.properties["classified_kind"] = classify_engineering_kind( + feat.properties + ) + logger.info( + "search_by_quarter layer=%s method=grid_walk count=%d quarter=%s", + name_in_dump, + len(features), + quarter_cad, + ) + return features + features_legacy = self.get_features_in_bbox(layer_id, bbox) + logger.info( + "search_by_quarter layer=%s method=legacy count=%d quarter=%s", + name_in_dump, + len(features_legacy), + quarter_cad, + ) + return features_legacy # 3. Core layers parcels = _fetch_layer("parcels", "parcels") diff --git a/backend/app/workers/tasks/nspd_sync.py b/backend/app/workers/tasks/nspd_sync.py index 3c377919..bb49674c 100644 --- a/backend/app/workers/tasks/nspd_sync.py +++ b/backend/app/workers/tasks/nspd_sync.py @@ -258,7 +258,7 @@ def _upsert_dump( bind=True, name="tasks.nspd_sync.harvest_quarter", max_retries=3, - soft_time_limit=120, # 2 мин — worst-case 22 HTTP × ~600ms = ~13s + margin + soft_time_limit=600, # 10 мин — grid-walk: 11 layers × 49 cells × ~70ms ≈ 40s + retries autoretry_for=(NspdLiteWafError,), retry_backoff=True, retry_backoff_max=120, diff --git a/backend/tests/scrapers/test_nspd_grid_walk.py b/backend/tests/scrapers/test_nspd_grid_walk.py index 25bdfec6..377ba20c 100644 --- a/backend/tests/scrapers/test_nspd_grid_walk.py +++ b/backend/tests/scrapers/test_nspd_grid_walk.py @@ -11,7 +11,7 @@ Live integration (помечены @pytest.mark.integration — пропуска from __future__ import annotations -from typing import Any +from typing import Any, ClassVar from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -301,6 +301,139 @@ class TestClassifyEngineeringKind: assert classify_engineering_kind({"name": "канализация бытовая"}) == "sewage" +# ── Sub-PR B: _fetch_layer dispatch tests ───────────────────────────────────── + + +class TestFetchLayerDispatch: + """Проверяем что _fetch_layer внутри search_by_quarter правильно диспатчит + area-слои на get_features_in_bbox_grid, а EGRN-слои — на legacy get_features_in_bbox. + + Моки: search_by_cad и get_features_in_bbox_grid / get_features_in_bbox + через patch, без live HTTP. + """ + + # Минимальный bbox квартала (Web Mercator), который возвращает search_by_cad + _MOCK_GEOM: ClassVar[dict[str, Any]] = { + "type": "Polygon", + "coordinates": [ + [ + [6_600_000.0, 7_700_000.0], + [6_600_700.0, 7_700_000.0], + [6_600_700.0, 7_700_700.0], + [6_600_000.0, 7_700_700.0], + [6_600_000.0, 7_700_000.0], + ] + ], + } + + def _make_search_result(self) -> MagicMock: + """Mock NSPDSearchResult с первым feature, у которого есть geometry.""" + feat = MagicMock(spec=NSPDFeature) + feat.geometry = self._MOCK_GEOM + feat.properties = {} + feat.feature_id = "q-001" + result = MagicMock() + result.first = feat + return result + + def test_fetch_layer_uses_grid_walk_for_area_layers(self) -> None: + """territorial_zones (area layer) → вызывает get_features_in_bbox_grid.""" + client = NSPDClient() + mock_search_result = self._make_search_result() + + with ( + patch.object(client, "search_by_cad", return_value=mock_search_result), + patch.object(client, "get_features_in_bbox_grid", return_value=[]) as mock_grid, + patch.object(client, "get_features_in_bbox", return_value=[]) as mock_legacy, + ): + client.search_by_quarter( + "66:41:0303161", + include_zouit=False, + include_risks=False, + include_opportunity=False, + ) + + # territorial_zones, red_lines, engineering_structures — все три должны + # быть запрошены через grid-walk + called_layer_ids = [call.args[0] for call in mock_grid.call_args_list] + from app.services.scrapers.nspd_client import LAYERS + + assert ( + LAYERS["territorial_zones"] in called_layer_ids + ), "territorial_zones должен использовать grid-walk" + assert LAYERS["red_lines"] in called_layer_ids, "red_lines должен использовать grid-walk" + assert ( + LAYERS["engineering_structures"] in called_layer_ids + ), "engineering_structures должен использовать grid-walk" + # parcels и buildings — legacy, не grid + called_legacy_ids = [call.args[0] for call in mock_legacy.call_args_list] + assert LAYERS["parcels"] in called_legacy_ids, "parcels должен идти через legacy" + assert LAYERS["buildings"] in called_legacy_ids, "buildings должен идти через legacy" + + def test_fetch_layer_uses_legacy_for_egrn_layers(self) -> None: + """parcels и buildings → вызывают legacy get_features_in_bbox (не grid).""" + client = NSPDClient() + mock_search_result = self._make_search_result() + + with ( + patch.object(client, "search_by_cad", return_value=mock_search_result), + patch.object(client, "get_features_in_bbox_grid", return_value=[]) as mock_grid, + patch.object(client, "get_features_in_bbox", return_value=[]) as mock_legacy, + ): + client.search_by_quarter( + "66:41:0303161", + include_zouit=False, + include_risks=False, + include_opportunity=False, + ) + + from app.services.scrapers.nspd_client import LAYERS + + called_legacy_ids = [call.args[0] for call in mock_legacy.call_args_list] + called_grid_ids = [call.args[0] for call in mock_grid.call_args_list] + + assert LAYERS["parcels"] in called_legacy_ids + assert LAYERS["buildings"] in called_legacy_ids + # EGRN layers должны НЕ попасть в grid-walk + assert LAYERS["parcels"] not in called_grid_ids, "parcels не должен использовать grid" + assert LAYERS["buildings"] not in called_grid_ids, "buildings не должен использовать grid" + + def test_engineering_structures_classified_kind_enriched(self) -> None: + """engineering_structures features получают classified_kind в properties.""" + from app.services.scrapers.nspd_client import LAYERS + + # Создаём feature с mutable properties (как реальный NSPDFeature) + eng_feat = MagicMock(spec=NSPDFeature) + eng_feat.feature_id = "eng-001" + eng_feat.geometry = None + eng_feat.properties = {"params_name": "Газопровод высокого давления"} + + client = NSPDClient() + mock_search_result = self._make_search_result() + + def _grid_side_effect(layer_id: int, bbox: Any, **kwargs: Any) -> list[Any]: + if layer_id == LAYERS["engineering_structures"]: + return [eng_feat] + return [] + + with ( + patch.object(client, "search_by_cad", return_value=mock_search_result), + patch.object(client, "get_features_in_bbox_grid", side_effect=_grid_side_effect), + patch.object(client, "get_features_in_bbox", return_value=[]), + ): + dump = client.search_by_quarter( + "66:41:0303161", + include_zouit=False, + include_risks=False, + include_opportunity=False, + ) + + assert len(dump.engineering_structures) == 1 + enriched = dump.engineering_structures[0] + got = enriched.properties.get("classified_kind") + assert got == "gas", f"Ожидали classified_kind='gas', получили {got!r}" + + # ── Integration marker (skip in CI) ──────────────────────────────────────────