fix(#231): switch search_by_quarter area layers to grid-walk dispatch (Sub-PR B)
- Add _GRID_WALK_LAYERS frozenset (territorial_zones, red_lines, engineering_structures, zouit_*, risk_*) - _fetch_layer dispatch: area layers → get_features_in_bbox_grid(grid_n=7, step_m=50) [foundation из PR #247], EGRN layers (parcels, buildings) → legacy get_features_in_bbox - Enrich engineering_structures features с classified_kind через classify_engineering_kind (PR #247 classifier) - Per-layer logger.info heartbeat (method + count) для прогресса - Bump harvest_quarter soft_time_limit 120 → 600s (grid-walk: 11 layers × 49 cells × ~70ms ≈ 40s + retries margin) - TestFetchLayerDispatch (3 tests): 34 passed, 1 skipped Pre-push deep-code-reviewer: APPROVE (1 medium nit — opportunity_oopt не в grid-walk, защищено через include_opportunity=False default; follow-up issue рекомендован).
This commit is contained in:
parent
9dd72ece0c
commit
fdced94626
3 changed files with 201 additions and 4 deletions
|
|
@ -43,6 +43,7 @@ import urllib.request
|
|||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from app.services.scrapers.nspd_denorm import classify_engineering_kind
|
||||
from app.services.scrapers.nspd_lite import (
|
||||
_SSL_CTX,
|
||||
HEADERS,
|
||||
|
|
@ -110,6 +111,37 @@ LAYERS: dict[str, int] = {
|
|||
}
|
||||
|
||||
|
||||
# Layers where grid-walk (get_features_in_bbox_grid) must be used instead of
|
||||
# the legacy single-pixel WMS probe (get_features_in_bbox). These are area/
|
||||
# linear layers (territorial zones, red lines, engineering structures, ЗОУИТ,
|
||||
# risks) that span large areas and are under-returned by a single GetFeatureInfo
|
||||
# call. Point/polygon EGRN layers (parcels, buildings) stay on legacy for now.
|
||||
_GRID_WALK_LAYERS: frozenset[str] = frozenset(
|
||||
{
|
||||
"territorial_zones",
|
||||
"red_lines",
|
||||
"engineering_structures",
|
||||
# ЗОУИТ (TIER 2)
|
||||
"zouit_okn",
|
||||
"zouit_engineering",
|
||||
"zouit_natural",
|
||||
"zouit_protected",
|
||||
"zouit_other",
|
||||
# Risks (TIER 3)
|
||||
"risk_flooding_underground",
|
||||
"risk_flooding",
|
||||
"risk_swampification",
|
||||
"risk_landslide",
|
||||
"risk_abrasion",
|
||||
"risk_erosion_water",
|
||||
"risk_erosion_linear",
|
||||
"risk_erosion_wind",
|
||||
"risk_desertification",
|
||||
"risk_clutter",
|
||||
"risk_burns",
|
||||
}
|
||||
)
|
||||
|
||||
# Default rate limit (мс между запросами) — баланс между скоростью и WAF
|
||||
DEFAULT_RATE_MS = 600
|
||||
|
||||
|
|
@ -709,7 +741,17 @@ class NSPDClient:
|
|||
layers_fetched: list[str] = ["search"]
|
||||
|
||||
def _fetch_layer(name_in_dump: str, layer_key: str) -> list[NSPDFeature]:
|
||||
"""Helper: безопасно получить features для одного layer."""
|
||||
"""Helper: безопасно получить features для одного layer.
|
||||
|
||||
Dispatch:
|
||||
- area/linear layers (_GRID_WALK_LAYERS) → grid-walk
|
||||
(get_features_in_bbox_grid, grid_n=7, step_m=50)
|
||||
- point/polygon EGRN layers (parcels, buildings, ons, enk)
|
||||
→ legacy single-pixel WMS (get_features_in_bbox)
|
||||
|
||||
Engineering structures features дополнительно обогащаются
|
||||
properties["classified_kind"] через classify_engineering_kind.
|
||||
"""
|
||||
if bbox is None:
|
||||
return []
|
||||
layer_id = LAYERS.get(layer_key)
|
||||
|
|
@ -717,7 +759,29 @@ class NSPDClient:
|
|||
logger.warning("search_by_quarter: unknown layer key %s", layer_key)
|
||||
return []
|
||||
layers_fetched.append(name_in_dump)
|
||||
return self.get_features_in_bbox(layer_id, bbox)
|
||||
if layer_key in _GRID_WALK_LAYERS:
|
||||
features = self.get_features_in_bbox_grid(layer_id, bbox, grid_n=7, step_m=50.0)
|
||||
# Обогатить engineering_structures classified_kind
|
||||
if layer_key == "engineering_structures":
|
||||
for feat in features:
|
||||
feat.properties["classified_kind"] = classify_engineering_kind(
|
||||
feat.properties
|
||||
)
|
||||
logger.info(
|
||||
"search_by_quarter layer=%s method=grid_walk count=%d quarter=%s",
|
||||
name_in_dump,
|
||||
len(features),
|
||||
quarter_cad,
|
||||
)
|
||||
return features
|
||||
features_legacy = self.get_features_in_bbox(layer_id, bbox)
|
||||
logger.info(
|
||||
"search_by_quarter layer=%s method=legacy count=%d quarter=%s",
|
||||
name_in_dump,
|
||||
len(features_legacy),
|
||||
quarter_cad,
|
||||
)
|
||||
return features_legacy
|
||||
|
||||
# 3. Core layers
|
||||
parcels = _fetch_layer("parcels", "parcels")
|
||||
|
|
|
|||
|
|
@ -258,7 +258,7 @@ def _upsert_dump(
|
|||
bind=True,
|
||||
name="tasks.nspd_sync.harvest_quarter",
|
||||
max_retries=3,
|
||||
soft_time_limit=120, # 2 мин — worst-case 22 HTTP × ~600ms = ~13s + margin
|
||||
soft_time_limit=600, # 10 мин — grid-walk: 11 layers × 49 cells × ~70ms ≈ 40s + retries
|
||||
autoretry_for=(NspdLiteWafError,),
|
||||
retry_backoff=True,
|
||||
retry_backoff_max=120,
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ Live integration (помечены @pytest.mark.integration — пропуска
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from typing import Any, ClassVar
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -301,6 +301,139 @@ class TestClassifyEngineeringKind:
|
|||
assert classify_engineering_kind({"name": "канализация бытовая"}) == "sewage"
|
||||
|
||||
|
||||
# ── Sub-PR B: _fetch_layer dispatch tests ─────────────────────────────────────
|
||||
|
||||
|
||||
class TestFetchLayerDispatch:
|
||||
"""Проверяем что _fetch_layer внутри search_by_quarter правильно диспатчит
|
||||
area-слои на get_features_in_bbox_grid, а EGRN-слои — на legacy get_features_in_bbox.
|
||||
|
||||
Моки: search_by_cad и get_features_in_bbox_grid / get_features_in_bbox
|
||||
через patch, без live HTTP.
|
||||
"""
|
||||
|
||||
# Минимальный bbox квартала (Web Mercator), который возвращает search_by_cad
|
||||
_MOCK_GEOM: ClassVar[dict[str, Any]] = {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[
|
||||
[6_600_000.0, 7_700_000.0],
|
||||
[6_600_700.0, 7_700_000.0],
|
||||
[6_600_700.0, 7_700_700.0],
|
||||
[6_600_000.0, 7_700_700.0],
|
||||
[6_600_000.0, 7_700_000.0],
|
||||
]
|
||||
],
|
||||
}
|
||||
|
||||
def _make_search_result(self) -> MagicMock:
|
||||
"""Mock NSPDSearchResult с первым feature, у которого есть geometry."""
|
||||
feat = MagicMock(spec=NSPDFeature)
|
||||
feat.geometry = self._MOCK_GEOM
|
||||
feat.properties = {}
|
||||
feat.feature_id = "q-001"
|
||||
result = MagicMock()
|
||||
result.first = feat
|
||||
return result
|
||||
|
||||
def test_fetch_layer_uses_grid_walk_for_area_layers(self) -> None:
|
||||
"""territorial_zones (area layer) → вызывает get_features_in_bbox_grid."""
|
||||
client = NSPDClient()
|
||||
mock_search_result = self._make_search_result()
|
||||
|
||||
with (
|
||||
patch.object(client, "search_by_cad", return_value=mock_search_result),
|
||||
patch.object(client, "get_features_in_bbox_grid", return_value=[]) as mock_grid,
|
||||
patch.object(client, "get_features_in_bbox", return_value=[]) as mock_legacy,
|
||||
):
|
||||
client.search_by_quarter(
|
||||
"66:41:0303161",
|
||||
include_zouit=False,
|
||||
include_risks=False,
|
||||
include_opportunity=False,
|
||||
)
|
||||
|
||||
# territorial_zones, red_lines, engineering_structures — все три должны
|
||||
# быть запрошены через grid-walk
|
||||
called_layer_ids = [call.args[0] for call in mock_grid.call_args_list]
|
||||
from app.services.scrapers.nspd_client import LAYERS
|
||||
|
||||
assert (
|
||||
LAYERS["territorial_zones"] in called_layer_ids
|
||||
), "territorial_zones должен использовать grid-walk"
|
||||
assert LAYERS["red_lines"] in called_layer_ids, "red_lines должен использовать grid-walk"
|
||||
assert (
|
||||
LAYERS["engineering_structures"] in called_layer_ids
|
||||
), "engineering_structures должен использовать grid-walk"
|
||||
# parcels и buildings — legacy, не grid
|
||||
called_legacy_ids = [call.args[0] for call in mock_legacy.call_args_list]
|
||||
assert LAYERS["parcels"] in called_legacy_ids, "parcels должен идти через legacy"
|
||||
assert LAYERS["buildings"] in called_legacy_ids, "buildings должен идти через legacy"
|
||||
|
||||
def test_fetch_layer_uses_legacy_for_egrn_layers(self) -> None:
|
||||
"""parcels и buildings → вызывают legacy get_features_in_bbox (не grid)."""
|
||||
client = NSPDClient()
|
||||
mock_search_result = self._make_search_result()
|
||||
|
||||
with (
|
||||
patch.object(client, "search_by_cad", return_value=mock_search_result),
|
||||
patch.object(client, "get_features_in_bbox_grid", return_value=[]) as mock_grid,
|
||||
patch.object(client, "get_features_in_bbox", return_value=[]) as mock_legacy,
|
||||
):
|
||||
client.search_by_quarter(
|
||||
"66:41:0303161",
|
||||
include_zouit=False,
|
||||
include_risks=False,
|
||||
include_opportunity=False,
|
||||
)
|
||||
|
||||
from app.services.scrapers.nspd_client import LAYERS
|
||||
|
||||
called_legacy_ids = [call.args[0] for call in mock_legacy.call_args_list]
|
||||
called_grid_ids = [call.args[0] for call in mock_grid.call_args_list]
|
||||
|
||||
assert LAYERS["parcels"] in called_legacy_ids
|
||||
assert LAYERS["buildings"] in called_legacy_ids
|
||||
# EGRN layers должны НЕ попасть в grid-walk
|
||||
assert LAYERS["parcels"] not in called_grid_ids, "parcels не должен использовать grid"
|
||||
assert LAYERS["buildings"] not in called_grid_ids, "buildings не должен использовать grid"
|
||||
|
||||
def test_engineering_structures_classified_kind_enriched(self) -> None:
|
||||
"""engineering_structures features получают classified_kind в properties."""
|
||||
from app.services.scrapers.nspd_client import LAYERS
|
||||
|
||||
# Создаём feature с mutable properties (как реальный NSPDFeature)
|
||||
eng_feat = MagicMock(spec=NSPDFeature)
|
||||
eng_feat.feature_id = "eng-001"
|
||||
eng_feat.geometry = None
|
||||
eng_feat.properties = {"params_name": "Газопровод высокого давления"}
|
||||
|
||||
client = NSPDClient()
|
||||
mock_search_result = self._make_search_result()
|
||||
|
||||
def _grid_side_effect(layer_id: int, bbox: Any, **kwargs: Any) -> list[Any]:
|
||||
if layer_id == LAYERS["engineering_structures"]:
|
||||
return [eng_feat]
|
||||
return []
|
||||
|
||||
with (
|
||||
patch.object(client, "search_by_cad", return_value=mock_search_result),
|
||||
patch.object(client, "get_features_in_bbox_grid", side_effect=_grid_side_effect),
|
||||
patch.object(client, "get_features_in_bbox", return_value=[]),
|
||||
):
|
||||
dump = client.search_by_quarter(
|
||||
"66:41:0303161",
|
||||
include_zouit=False,
|
||||
include_risks=False,
|
||||
include_opportunity=False,
|
||||
)
|
||||
|
||||
assert len(dump.engineering_structures) == 1
|
||||
enriched = dump.engineering_structures[0]
|
||||
got = enriched.properties.get("classified_kind")
|
||||
assert got == "gas", f"Ожидали classified_kind='gas', получили {got!r}"
|
||||
|
||||
|
||||
# ── Integration marker (skip in CI) ──────────────────────────────────────────
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue