fix(#231): switch search_by_quarter area layers to grid-walk dispatch (Sub-PR B)
- Add _GRID_WALK_LAYERS frozenset (territorial_zones, red_lines, engineering_structures, zouit_*, risk_*) - _fetch_layer dispatch: area layers → get_features_in_bbox_grid(grid_n=7, step_m=50) [foundation из PR #247], EGRN layers (parcels, buildings) → legacy get_features_in_bbox - Enrich engineering_structures features с classified_kind через classify_engineering_kind (PR #247 classifier) - Per-layer logger.info heartbeat (method + count) для прогресса - Bump harvest_quarter soft_time_limit 120 → 600s (grid-walk: 11 layers × 49 cells × ~70ms ≈ 40s + retries margin) - TestFetchLayerDispatch (3 tests): 34 passed, 1 skipped Pre-push deep-code-reviewer: APPROVE (1 medium nit — opportunity_oopt не в grid-walk, защищено через include_opportunity=False default; follow-up issue рекомендован).
This commit is contained in:
parent
9dd72ece0c
commit
fdced94626
3 changed files with 201 additions and 4 deletions
|
|
@ -43,6 +43,7 @@ import urllib.request
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
from app.services.scrapers.nspd_denorm import classify_engineering_kind
|
||||||
from app.services.scrapers.nspd_lite import (
|
from app.services.scrapers.nspd_lite import (
|
||||||
_SSL_CTX,
|
_SSL_CTX,
|
||||||
HEADERS,
|
HEADERS,
|
||||||
|
|
@ -110,6 +111,37 @@ LAYERS: dict[str, int] = {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# Layers where grid-walk (get_features_in_bbox_grid) must be used instead of
|
||||||
|
# the legacy single-pixel WMS probe (get_features_in_bbox). These are area/
|
||||||
|
# linear layers (territorial zones, red lines, engineering structures, ЗОУИТ,
|
||||||
|
# risks) that span large areas and are under-returned by a single GetFeatureInfo
|
||||||
|
# call. Point/polygon EGRN layers (parcels, buildings) stay on legacy for now.
|
||||||
|
_GRID_WALK_LAYERS: frozenset[str] = frozenset(
|
||||||
|
{
|
||||||
|
"territorial_zones",
|
||||||
|
"red_lines",
|
||||||
|
"engineering_structures",
|
||||||
|
# ЗОУИТ (TIER 2)
|
||||||
|
"zouit_okn",
|
||||||
|
"zouit_engineering",
|
||||||
|
"zouit_natural",
|
||||||
|
"zouit_protected",
|
||||||
|
"zouit_other",
|
||||||
|
# Risks (TIER 3)
|
||||||
|
"risk_flooding_underground",
|
||||||
|
"risk_flooding",
|
||||||
|
"risk_swampification",
|
||||||
|
"risk_landslide",
|
||||||
|
"risk_abrasion",
|
||||||
|
"risk_erosion_water",
|
||||||
|
"risk_erosion_linear",
|
||||||
|
"risk_erosion_wind",
|
||||||
|
"risk_desertification",
|
||||||
|
"risk_clutter",
|
||||||
|
"risk_burns",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
# Default rate limit (мс между запросами) — баланс между скоростью и WAF
|
# Default rate limit (мс между запросами) — баланс между скоростью и WAF
|
||||||
DEFAULT_RATE_MS = 600
|
DEFAULT_RATE_MS = 600
|
||||||
|
|
||||||
|
|
@ -709,7 +741,17 @@ class NSPDClient:
|
||||||
layers_fetched: list[str] = ["search"]
|
layers_fetched: list[str] = ["search"]
|
||||||
|
|
||||||
def _fetch_layer(name_in_dump: str, layer_key: str) -> list[NSPDFeature]:
|
def _fetch_layer(name_in_dump: str, layer_key: str) -> list[NSPDFeature]:
|
||||||
"""Helper: безопасно получить features для одного layer."""
|
"""Helper: безопасно получить features для одного layer.
|
||||||
|
|
||||||
|
Dispatch:
|
||||||
|
- area/linear layers (_GRID_WALK_LAYERS) → grid-walk
|
||||||
|
(get_features_in_bbox_grid, grid_n=7, step_m=50)
|
||||||
|
- point/polygon EGRN layers (parcels, buildings, ons, enk)
|
||||||
|
→ legacy single-pixel WMS (get_features_in_bbox)
|
||||||
|
|
||||||
|
Engineering structures features дополнительно обогащаются
|
||||||
|
properties["classified_kind"] через classify_engineering_kind.
|
||||||
|
"""
|
||||||
if bbox is None:
|
if bbox is None:
|
||||||
return []
|
return []
|
||||||
layer_id = LAYERS.get(layer_key)
|
layer_id = LAYERS.get(layer_key)
|
||||||
|
|
@ -717,7 +759,29 @@ class NSPDClient:
|
||||||
logger.warning("search_by_quarter: unknown layer key %s", layer_key)
|
logger.warning("search_by_quarter: unknown layer key %s", layer_key)
|
||||||
return []
|
return []
|
||||||
layers_fetched.append(name_in_dump)
|
layers_fetched.append(name_in_dump)
|
||||||
return self.get_features_in_bbox(layer_id, bbox)
|
if layer_key in _GRID_WALK_LAYERS:
|
||||||
|
features = self.get_features_in_bbox_grid(layer_id, bbox, grid_n=7, step_m=50.0)
|
||||||
|
# Обогатить engineering_structures classified_kind
|
||||||
|
if layer_key == "engineering_structures":
|
||||||
|
for feat in features:
|
||||||
|
feat.properties["classified_kind"] = classify_engineering_kind(
|
||||||
|
feat.properties
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"search_by_quarter layer=%s method=grid_walk count=%d quarter=%s",
|
||||||
|
name_in_dump,
|
||||||
|
len(features),
|
||||||
|
quarter_cad,
|
||||||
|
)
|
||||||
|
return features
|
||||||
|
features_legacy = self.get_features_in_bbox(layer_id, bbox)
|
||||||
|
logger.info(
|
||||||
|
"search_by_quarter layer=%s method=legacy count=%d quarter=%s",
|
||||||
|
name_in_dump,
|
||||||
|
len(features_legacy),
|
||||||
|
quarter_cad,
|
||||||
|
)
|
||||||
|
return features_legacy
|
||||||
|
|
||||||
# 3. Core layers
|
# 3. Core layers
|
||||||
parcels = _fetch_layer("parcels", "parcels")
|
parcels = _fetch_layer("parcels", "parcels")
|
||||||
|
|
|
||||||
|
|
@ -258,7 +258,7 @@ def _upsert_dump(
|
||||||
bind=True,
|
bind=True,
|
||||||
name="tasks.nspd_sync.harvest_quarter",
|
name="tasks.nspd_sync.harvest_quarter",
|
||||||
max_retries=3,
|
max_retries=3,
|
||||||
soft_time_limit=120, # 2 мин — worst-case 22 HTTP × ~600ms = ~13s + margin
|
soft_time_limit=600, # 10 мин — grid-walk: 11 layers × 49 cells × ~70ms ≈ 40s + retries
|
||||||
autoretry_for=(NspdLiteWafError,),
|
autoretry_for=(NspdLiteWafError,),
|
||||||
retry_backoff=True,
|
retry_backoff=True,
|
||||||
retry_backoff_max=120,
|
retry_backoff_max=120,
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@ Live integration (помечены @pytest.mark.integration — пропуска
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from typing import Any
|
from typing import Any, ClassVar
|
||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
@ -301,6 +301,139 @@ class TestClassifyEngineeringKind:
|
||||||
assert classify_engineering_kind({"name": "канализация бытовая"}) == "sewage"
|
assert classify_engineering_kind({"name": "канализация бытовая"}) == "sewage"
|
||||||
|
|
||||||
|
|
||||||
|
# ── Sub-PR B: _fetch_layer dispatch tests ─────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestFetchLayerDispatch:
|
||||||
|
"""Проверяем что _fetch_layer внутри search_by_quarter правильно диспатчит
|
||||||
|
area-слои на get_features_in_bbox_grid, а EGRN-слои — на legacy get_features_in_bbox.
|
||||||
|
|
||||||
|
Моки: search_by_cad и get_features_in_bbox_grid / get_features_in_bbox
|
||||||
|
через patch, без live HTTP.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Минимальный bbox квартала (Web Mercator), который возвращает search_by_cad
|
||||||
|
_MOCK_GEOM: ClassVar[dict[str, Any]] = {
|
||||||
|
"type": "Polygon",
|
||||||
|
"coordinates": [
|
||||||
|
[
|
||||||
|
[6_600_000.0, 7_700_000.0],
|
||||||
|
[6_600_700.0, 7_700_000.0],
|
||||||
|
[6_600_700.0, 7_700_700.0],
|
||||||
|
[6_600_000.0, 7_700_700.0],
|
||||||
|
[6_600_000.0, 7_700_000.0],
|
||||||
|
]
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
def _make_search_result(self) -> MagicMock:
|
||||||
|
"""Mock NSPDSearchResult с первым feature, у которого есть geometry."""
|
||||||
|
feat = MagicMock(spec=NSPDFeature)
|
||||||
|
feat.geometry = self._MOCK_GEOM
|
||||||
|
feat.properties = {}
|
||||||
|
feat.feature_id = "q-001"
|
||||||
|
result = MagicMock()
|
||||||
|
result.first = feat
|
||||||
|
return result
|
||||||
|
|
||||||
|
def test_fetch_layer_uses_grid_walk_for_area_layers(self) -> None:
|
||||||
|
"""territorial_zones (area layer) → вызывает get_features_in_bbox_grid."""
|
||||||
|
client = NSPDClient()
|
||||||
|
mock_search_result = self._make_search_result()
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch.object(client, "search_by_cad", return_value=mock_search_result),
|
||||||
|
patch.object(client, "get_features_in_bbox_grid", return_value=[]) as mock_grid,
|
||||||
|
patch.object(client, "get_features_in_bbox", return_value=[]) as mock_legacy,
|
||||||
|
):
|
||||||
|
client.search_by_quarter(
|
||||||
|
"66:41:0303161",
|
||||||
|
include_zouit=False,
|
||||||
|
include_risks=False,
|
||||||
|
include_opportunity=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
# territorial_zones, red_lines, engineering_structures — все три должны
|
||||||
|
# быть запрошены через grid-walk
|
||||||
|
called_layer_ids = [call.args[0] for call in mock_grid.call_args_list]
|
||||||
|
from app.services.scrapers.nspd_client import LAYERS
|
||||||
|
|
||||||
|
assert (
|
||||||
|
LAYERS["territorial_zones"] in called_layer_ids
|
||||||
|
), "territorial_zones должен использовать grid-walk"
|
||||||
|
assert LAYERS["red_lines"] in called_layer_ids, "red_lines должен использовать grid-walk"
|
||||||
|
assert (
|
||||||
|
LAYERS["engineering_structures"] in called_layer_ids
|
||||||
|
), "engineering_structures должен использовать grid-walk"
|
||||||
|
# parcels и buildings — legacy, не grid
|
||||||
|
called_legacy_ids = [call.args[0] for call in mock_legacy.call_args_list]
|
||||||
|
assert LAYERS["parcels"] in called_legacy_ids, "parcels должен идти через legacy"
|
||||||
|
assert LAYERS["buildings"] in called_legacy_ids, "buildings должен идти через legacy"
|
||||||
|
|
||||||
|
def test_fetch_layer_uses_legacy_for_egrn_layers(self) -> None:
|
||||||
|
"""parcels и buildings → вызывают legacy get_features_in_bbox (не grid)."""
|
||||||
|
client = NSPDClient()
|
||||||
|
mock_search_result = self._make_search_result()
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch.object(client, "search_by_cad", return_value=mock_search_result),
|
||||||
|
patch.object(client, "get_features_in_bbox_grid", return_value=[]) as mock_grid,
|
||||||
|
patch.object(client, "get_features_in_bbox", return_value=[]) as mock_legacy,
|
||||||
|
):
|
||||||
|
client.search_by_quarter(
|
||||||
|
"66:41:0303161",
|
||||||
|
include_zouit=False,
|
||||||
|
include_risks=False,
|
||||||
|
include_opportunity=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
from app.services.scrapers.nspd_client import LAYERS
|
||||||
|
|
||||||
|
called_legacy_ids = [call.args[0] for call in mock_legacy.call_args_list]
|
||||||
|
called_grid_ids = [call.args[0] for call in mock_grid.call_args_list]
|
||||||
|
|
||||||
|
assert LAYERS["parcels"] in called_legacy_ids
|
||||||
|
assert LAYERS["buildings"] in called_legacy_ids
|
||||||
|
# EGRN layers должны НЕ попасть в grid-walk
|
||||||
|
assert LAYERS["parcels"] not in called_grid_ids, "parcels не должен использовать grid"
|
||||||
|
assert LAYERS["buildings"] not in called_grid_ids, "buildings не должен использовать grid"
|
||||||
|
|
||||||
|
def test_engineering_structures_classified_kind_enriched(self) -> None:
|
||||||
|
"""engineering_structures features получают classified_kind в properties."""
|
||||||
|
from app.services.scrapers.nspd_client import LAYERS
|
||||||
|
|
||||||
|
# Создаём feature с mutable properties (как реальный NSPDFeature)
|
||||||
|
eng_feat = MagicMock(spec=NSPDFeature)
|
||||||
|
eng_feat.feature_id = "eng-001"
|
||||||
|
eng_feat.geometry = None
|
||||||
|
eng_feat.properties = {"params_name": "Газопровод высокого давления"}
|
||||||
|
|
||||||
|
client = NSPDClient()
|
||||||
|
mock_search_result = self._make_search_result()
|
||||||
|
|
||||||
|
def _grid_side_effect(layer_id: int, bbox: Any, **kwargs: Any) -> list[Any]:
|
||||||
|
if layer_id == LAYERS["engineering_structures"]:
|
||||||
|
return [eng_feat]
|
||||||
|
return []
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch.object(client, "search_by_cad", return_value=mock_search_result),
|
||||||
|
patch.object(client, "get_features_in_bbox_grid", side_effect=_grid_side_effect),
|
||||||
|
patch.object(client, "get_features_in_bbox", return_value=[]),
|
||||||
|
):
|
||||||
|
dump = client.search_by_quarter(
|
||||||
|
"66:41:0303161",
|
||||||
|
include_zouit=False,
|
||||||
|
include_risks=False,
|
||||||
|
include_opportunity=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert len(dump.engineering_structures) == 1
|
||||||
|
enriched = dump.engineering_structures[0]
|
||||||
|
got = enriched.properties.get("classified_kind")
|
||||||
|
assert got == "gas", f"Ожидали classified_kind='gas', получили {got!r}"
|
||||||
|
|
||||||
|
|
||||||
# ── Integration marker (skip in CI) ──────────────────────────────────────────
|
# ── Integration marker (skip in CI) ──────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue