fix(#231): switch search_by_quarter area layers to grid-walk dispatch (Sub-PR B)

- Add _GRID_WALK_LAYERS frozenset (territorial_zones, red_lines,
  engineering_structures, zouit_*, risk_*)
- _fetch_layer dispatch: area layers → get_features_in_bbox_grid(grid_n=7,
  step_m=50) [foundation из PR #247], EGRN layers (parcels, buildings) →
  legacy get_features_in_bbox
- Enrich engineering_structures features с classified_kind через
  classify_engineering_kind (PR #247 classifier)
- Per-layer logger.info heartbeat (method + count) для прогресса
- Bump harvest_quarter soft_time_limit 120 → 600s
  (grid-walk: 11 layers × 49 cells × ~70ms ≈ 40s + retries margin)
- TestFetchLayerDispatch (3 tests): 34 passed, 1 skipped

Pre-push deep-code-reviewer: APPROVE (1 medium nit — opportunity_oopt не в
grid-walk, защищено через include_opportunity=False default; follow-up issue
рекомендован).
This commit is contained in:
lekss361 2026-05-17 09:46:42 +03:00
parent 9dd72ece0c
commit fdced94626
3 changed files with 201 additions and 4 deletions

View file

@ -43,6 +43,7 @@ import urllib.request
from dataclasses import dataclass
from typing import Any
from app.services.scrapers.nspd_denorm import classify_engineering_kind
from app.services.scrapers.nspd_lite import (
_SSL_CTX,
HEADERS,
@ -110,6 +111,37 @@ LAYERS: dict[str, int] = {
}
# Layers where grid-walk (get_features_in_bbox_grid) must be used instead of
# the legacy single-pixel WMS probe (get_features_in_bbox). These are area/
# linear layers (territorial zones, red lines, engineering structures, ЗОУИТ,
# risks) that span large areas and are under-returned by a single GetFeatureInfo
# call. Point/polygon EGRN layers (parcels, buildings) stay on legacy for now.
_GRID_WALK_LAYERS: frozenset[str] = frozenset(
{
"territorial_zones",
"red_lines",
"engineering_structures",
# ЗОУИТ (TIER 2)
"zouit_okn",
"zouit_engineering",
"zouit_natural",
"zouit_protected",
"zouit_other",
# Risks (TIER 3)
"risk_flooding_underground",
"risk_flooding",
"risk_swampification",
"risk_landslide",
"risk_abrasion",
"risk_erosion_water",
"risk_erosion_linear",
"risk_erosion_wind",
"risk_desertification",
"risk_clutter",
"risk_burns",
}
)
# Default rate limit (мс между запросами) — баланс между скоростью и WAF
DEFAULT_RATE_MS = 600
@ -709,7 +741,17 @@ class NSPDClient:
layers_fetched: list[str] = ["search"]
def _fetch_layer(name_in_dump: str, layer_key: str) -> list[NSPDFeature]:
"""Helper: безопасно получить features для одного layer."""
"""Helper: безопасно получить features для одного layer.
Dispatch:
- area/linear layers (_GRID_WALK_LAYERS) grid-walk
(get_features_in_bbox_grid, grid_n=7, step_m=50)
- point/polygon EGRN layers (parcels, buildings, ons, enk)
legacy single-pixel WMS (get_features_in_bbox)
Engineering structures features дополнительно обогащаются
properties["classified_kind"] через classify_engineering_kind.
"""
if bbox is None:
return []
layer_id = LAYERS.get(layer_key)
@ -717,7 +759,29 @@ class NSPDClient:
logger.warning("search_by_quarter: unknown layer key %s", layer_key)
return []
layers_fetched.append(name_in_dump)
return self.get_features_in_bbox(layer_id, bbox)
if layer_key in _GRID_WALK_LAYERS:
features = self.get_features_in_bbox_grid(layer_id, bbox, grid_n=7, step_m=50.0)
# Обогатить engineering_structures classified_kind
if layer_key == "engineering_structures":
for feat in features:
feat.properties["classified_kind"] = classify_engineering_kind(
feat.properties
)
logger.info(
"search_by_quarter layer=%s method=grid_walk count=%d quarter=%s",
name_in_dump,
len(features),
quarter_cad,
)
return features
features_legacy = self.get_features_in_bbox(layer_id, bbox)
logger.info(
"search_by_quarter layer=%s method=legacy count=%d quarter=%s",
name_in_dump,
len(features_legacy),
quarter_cad,
)
return features_legacy
# 3. Core layers
parcels = _fetch_layer("parcels", "parcels")

View file

@ -258,7 +258,7 @@ def _upsert_dump(
bind=True,
name="tasks.nspd_sync.harvest_quarter",
max_retries=3,
soft_time_limit=120, # 2 мин — worst-case 22 HTTP × ~600ms = ~13s + margin
soft_time_limit=600, # 10 мин — grid-walk: 11 layers × 49 cells × ~70ms ≈ 40s + retries
autoretry_for=(NspdLiteWafError,),
retry_backoff=True,
retry_backoff_max=120,

View file

@ -11,7 +11,7 @@ Live integration (помечены @pytest.mark.integration — пропуска
from __future__ import annotations
from typing import Any
from typing import Any, ClassVar
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@ -301,6 +301,139 @@ class TestClassifyEngineeringKind:
assert classify_engineering_kind({"name": "канализация бытовая"}) == "sewage"
# ── Sub-PR B: _fetch_layer dispatch tests ─────────────────────────────────────
class TestFetchLayerDispatch:
"""Проверяем что _fetch_layer внутри search_by_quarter правильно диспатчит
area-слои на get_features_in_bbox_grid, а EGRN-слои на legacy get_features_in_bbox.
Моки: search_by_cad и get_features_in_bbox_grid / get_features_in_bbox
через patch, без live HTTP.
"""
# Минимальный bbox квартала (Web Mercator), который возвращает search_by_cad
_MOCK_GEOM: ClassVar[dict[str, Any]] = {
"type": "Polygon",
"coordinates": [
[
[6_600_000.0, 7_700_000.0],
[6_600_700.0, 7_700_000.0],
[6_600_700.0, 7_700_700.0],
[6_600_000.0, 7_700_700.0],
[6_600_000.0, 7_700_000.0],
]
],
}
def _make_search_result(self) -> MagicMock:
"""Mock NSPDSearchResult с первым feature, у которого есть geometry."""
feat = MagicMock(spec=NSPDFeature)
feat.geometry = self._MOCK_GEOM
feat.properties = {}
feat.feature_id = "q-001"
result = MagicMock()
result.first = feat
return result
def test_fetch_layer_uses_grid_walk_for_area_layers(self) -> None:
"""territorial_zones (area layer) → вызывает get_features_in_bbox_grid."""
client = NSPDClient()
mock_search_result = self._make_search_result()
with (
patch.object(client, "search_by_cad", return_value=mock_search_result),
patch.object(client, "get_features_in_bbox_grid", return_value=[]) as mock_grid,
patch.object(client, "get_features_in_bbox", return_value=[]) as mock_legacy,
):
client.search_by_quarter(
"66:41:0303161",
include_zouit=False,
include_risks=False,
include_opportunity=False,
)
# territorial_zones, red_lines, engineering_structures — все три должны
# быть запрошены через grid-walk
called_layer_ids = [call.args[0] for call in mock_grid.call_args_list]
from app.services.scrapers.nspd_client import LAYERS
assert (
LAYERS["territorial_zones"] in called_layer_ids
), "territorial_zones должен использовать grid-walk"
assert LAYERS["red_lines"] in called_layer_ids, "red_lines должен использовать grid-walk"
assert (
LAYERS["engineering_structures"] in called_layer_ids
), "engineering_structures должен использовать grid-walk"
# parcels и buildings — legacy, не grid
called_legacy_ids = [call.args[0] for call in mock_legacy.call_args_list]
assert LAYERS["parcels"] in called_legacy_ids, "parcels должен идти через legacy"
assert LAYERS["buildings"] in called_legacy_ids, "buildings должен идти через legacy"
def test_fetch_layer_uses_legacy_for_egrn_layers(self) -> None:
"""parcels и buildings → вызывают legacy get_features_in_bbox (не grid)."""
client = NSPDClient()
mock_search_result = self._make_search_result()
with (
patch.object(client, "search_by_cad", return_value=mock_search_result),
patch.object(client, "get_features_in_bbox_grid", return_value=[]) as mock_grid,
patch.object(client, "get_features_in_bbox", return_value=[]) as mock_legacy,
):
client.search_by_quarter(
"66:41:0303161",
include_zouit=False,
include_risks=False,
include_opportunity=False,
)
from app.services.scrapers.nspd_client import LAYERS
called_legacy_ids = [call.args[0] for call in mock_legacy.call_args_list]
called_grid_ids = [call.args[0] for call in mock_grid.call_args_list]
assert LAYERS["parcels"] in called_legacy_ids
assert LAYERS["buildings"] in called_legacy_ids
# EGRN layers должны НЕ попасть в grid-walk
assert LAYERS["parcels"] not in called_grid_ids, "parcels не должен использовать grid"
assert LAYERS["buildings"] not in called_grid_ids, "buildings не должен использовать grid"
def test_engineering_structures_classified_kind_enriched(self) -> None:
"""engineering_structures features получают classified_kind в properties."""
from app.services.scrapers.nspd_client import LAYERS
# Создаём feature с mutable properties (как реальный NSPDFeature)
eng_feat = MagicMock(spec=NSPDFeature)
eng_feat.feature_id = "eng-001"
eng_feat.geometry = None
eng_feat.properties = {"params_name": "Газопровод высокого давления"}
client = NSPDClient()
mock_search_result = self._make_search_result()
def _grid_side_effect(layer_id: int, bbox: Any, **kwargs: Any) -> list[Any]:
if layer_id == LAYERS["engineering_structures"]:
return [eng_feat]
return []
with (
patch.object(client, "search_by_cad", return_value=mock_search_result),
patch.object(client, "get_features_in_bbox_grid", side_effect=_grid_side_effect),
patch.object(client, "get_features_in_bbox", return_value=[]),
):
dump = client.search_by_quarter(
"66:41:0303161",
include_zouit=False,
include_risks=False,
include_opportunity=False,
)
assert len(dump.engineering_structures) == 1
enriched = dump.engineering_structures[0]
got = enriched.properties.get("classified_kind")
assert got == "gas", f"Ожидали classified_kind='gas', получили {got!r}"
# ── Integration marker (skip in CI) ──────────────────────────────────────────