Merge remote-tracking branch 'forgejo/main' into fix/234-nspd-harvest-eta-badge

# Conflicts:
#	backend/app/services/site_finder/quarter_dump_lookup.py
This commit is contained in:
lekss361 2026-05-17 09:19:12 +03:00
commit b99ac76463
9 changed files with 811 additions and 21 deletions

View file

@ -248,10 +248,13 @@ class EkburgPermitsClient:
USER_AGENT = "GenDesign/1.0 (+https://gendsgn.ru) Site Finder permits scraper" USER_AGENT = "GenDesign/1.0 (+https://gendsgn.ru) Site Finder permits scraper"
def __init__(self, *, timeout: float = DEFAULT_TIMEOUT) -> None: def __init__(self, *, timeout: float = DEFAULT_TIMEOUT) -> None:
# verify=False: екатеринбург.рф подписан CA Минцифры РФ (нет в certifi).
# Данные публичные open-data — SSL pinning здесь не требуется. Issue #242.
self._client = httpx.Client( self._client = httpx.Client(
timeout=timeout, timeout=timeout,
follow_redirects=True, follow_redirects=True,
headers={"User-Agent": self.USER_AGENT}, headers={"User-Agent": self.USER_AGENT},
verify=False,
) )
def __enter__(self) -> EkburgPermitsClient: def __enter__(self) -> EkburgPermitsClient:

View file

@ -31,6 +31,7 @@ WMS endpoints (per #94 issue body, TIER 1-6 каталог слоёв):
from __future__ import annotations from __future__ import annotations
import asyncio
import datetime as _dt import datetime as _dt
import json import json
import logging import logging
@ -384,24 +385,22 @@ class NSPDClient:
width: int = 4096, width: int = 4096,
height: int = 4096, height: int = 4096,
) -> list[NSPDFeature]: ) -> list[NSPDFeature]:
"""Bulk fetch features в bbox через GetFeatureInfo с большим bbox. """WMS GetFeatureInfo на одном центральном пикселе bbox.
Workaround: WFS GetCapabilities 404 на nspd.gov.ru, нет WFS DEPRECATED: возвращает 0-3 features под одним пикселем (I=W/2, J=H/2).
GetFeature endpoint. Решение: использовать GetFeatureInfo с large НЕ является bulk fetch несмотря на исходный docstring WMS GetFeatureInfo
bbox и точкой в центре (I=W/2, J=H/2) возвращает все features по стандарту OGC возвращает объекты строго под одной pixel-точкой, а не
пересекающиеся с bbox. во всём bbox. Для получения всех объектов в bbox используй
`get_features_in_bbox_grid`.
See: fixes/Bug_NSPD_WMS_NotBulk_2026_May14.md
Args: Args:
bbox_3857: (xmin, ymin, xmax, ymax) в EPSG:3857 метрах. bbox_3857: (xmin, ymin, xmax, ymax) в EPSG:3857 метрах.
width/height: размер виртуального tile. Большой большой bbox. width/height: размер виртуального tile.
Returns: Returns:
list[NSPDFeature]; пусто если ничего не найдено. list[NSPDFeature]; пусто если ничего не найдено.
Use cases (per #94 acceptance):
- sync_territorial_zones_bbox закрывает G1 #28 ПЗЗ
- sync_zouit_*_bbox G3 #30
- sync_risk_zones_bbox новый risk overlay
""" """
xmin, ymin, xmax, ymax = bbox_3857 xmin, ymin, xmax, ymax = bbox_3857
params = { params = {
@ -426,6 +425,149 @@ class NSPDClient:
feats = (data or {}).get("features") or [] feats = (data or {}).get("features") or []
return [NSPDFeature.from_raw(f) for f in feats] return [NSPDFeature.from_raw(f) for f in feats]
# ── 3b. get_features_in_bbox_grid ───────────────────────────────────────
def get_features_in_bbox_grid(
self,
layer_id: int,
bbox: tuple[float, float, float, float],
*,
grid_n: int = 7,
step_m: float = 50.0,
tile_size: int = 512,
) -> list[NSPDFeature]:
"""Bulk-аппроксимация bbox через grid-walk WMS GetFeatureInfo.
Разбивает bbox на grid_n × grid_n равных ячеек. В каждой ячейке
вызывает WMS GetFeatureInfo в центральном пикселе. Дедуплицирует
результаты по feature_id / cad_num / reg_numb_border возвращает
список уникальных NSPDFeature.
Делегирует HTTP через NSPDBulkClient.wms_feature_info (async httpx
с semaphore и retry), запуская asyncio event loop синхронно через
asyncio.run(). Предназначен для вызова из синхронного кода (Celery
task, FastAPI sync handler).
Concurrency: NSPDBulkClient._SEMAPHORE(3) ограничивает параллельные
запросы. При grid_n=7 (49 ячеек) все 49 ячеек запускаются одним
gather; семафор пропускает не более 3 одновременно. Thread-safety:
каждый вызов get_features_in_bbox_grid создаёт новый event loop
через asyncio.run() безопасно из разных Celery workers (process-
уровень изоляции).
Args:
layer_id: NSPD layer ID (например 36328 сооружения, 37578 ЗОУИТ).
bbox: (xmin, ymin, xmax, ymax) в EPSG:3857 (метры).
grid_n: размер сетки по каждой оси. 7 49 запросов (~coarse),
15 225 запросов (~fine). По умолчанию 7 для первичного scan.
step_m: минимальный шаг ячейки в метрах. Если bbox меньше
grid_n*step_m grid_n уменьшается автоматически чтобы
ячейки не становились меньше step_m.
tile_size: размер виртуального WMS тайла (пиксели).
Returns:
Дедуплицированный list[NSPDFeature]. Может быть пуст если в bbox
нет объектов данного layer'а.
Note:
Не делает live HTTP вызовы если вызван с mock NSPDBulkClient.
Rate-limit управляется семафором NSPDBulkClient._SEMAPHORE(3) +
asyncio.sleep(0.05) jitter не через self.rate_ms.
"""
# Импортируем здесь чтобы избежать circular import:
# nspd_client ← nspd_bulk_client (оба top-level scrapers, не cross-domain)
from app.scrapers.nspd_bulk_client import NSPDBulkClient
xmin, ymin, xmax, ymax = bbox
width_m = xmax - xmin
height_m = ymax - ymin
# Авто-коррекция grid_n если bbox мал для шага step_m
effective_n = min(
grid_n,
max(1, int(width_m / step_m)),
max(1, int(height_m / step_m)),
)
if effective_n < grid_n:
logger.info(
"get_features_in_bbox_grid layer=%d: bbox %.0fx%.0fm < grid_n=%d×step_m=%.0f"
" — уменьшаем grid до %d×%d",
layer_id,
width_m,
height_m,
grid_n,
step_m,
effective_n,
effective_n,
)
x_step = width_m / effective_n
y_step = height_m / effective_n
# Генерируем список (sub_bbox, click_xy) ячеек
cells: list[tuple[tuple[float, float, float, float], tuple[int, int]]] = []
click_px = tile_size // 2
for i in range(effective_n):
for j in range(effective_n):
cell_xmin = xmin + i * x_step
cell_ymin = ymin + j * y_step
cell_xmax = cell_xmin + x_step
cell_ymax = cell_ymin + y_step
cells.append(((cell_xmin, cell_ymin, cell_xmax, cell_ymax), (click_px, click_px)))
async def _run_grid() -> list[NSPDFeature]:
async with NSPDBulkClient() as client:
tasks = [
client.wms_feature_info(layer_id, sub_bbox, click_xy, tile_size, tile_size)
for sub_bbox, click_xy in cells
]
results = await asyncio.gather(*tasks, return_exceptions=True)
features: list[NSPDFeature] = []
for r in results:
if isinstance(r, Exception):
logger.warning("get_features_in_bbox_grid layer=%d cell error: %s", layer_id, r)
continue
for bulk_feat in r:
raw = {
"id": bulk_feat.id,
"geometry": bulk_feat.geometry,
"properties": bulk_feat.properties,
}
features.append(NSPDFeature.from_raw(raw))
return features
raw_features = asyncio.run(_run_grid())
# Дедупликация — приоритет ключей: feature_id > cad_num > reg_numb_border
seen: set[str] = set()
deduped: list[NSPDFeature] = []
for feat in raw_features:
props = feat.properties
dedup_key = (
feat.feature_id
or props.get("cad_num")
or props.get("cad_number")
or props.get("reg_numb_border")
or props.get("label")
)
if dedup_key is not None:
if dedup_key in seen:
continue
seen.add(dedup_key)
deduped.append(feat)
logger.info(
"get_features_in_bbox_grid layer=%d grid=%dx%d cells=%d raw=%d deduped=%d",
layer_id,
effective_n,
effective_n,
len(cells),
len(raw_features),
len(deduped),
)
return deduped
# ── 4. list_layers ────────────────────────────────────────────────────── # ── 4. list_layers ──────────────────────────────────────────────────────
def list_layers(self, theme_id: int = THEME_PKK) -> list[NSPDLayer]: def list_layers(self, theme_id: int = THEME_PKK) -> list[NSPDLayer]:

View file

@ -32,6 +32,7 @@ from __future__ import annotations
import datetime import datetime
import json import json
import logging import logging
import re
from typing import Any from typing import Any
from sqlalchemy import text from sqlalchemy import text
@ -39,6 +40,84 @@ from sqlalchemy.orm import Session
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# ── Engineering classifier ─────────────────────────────────────────────────────
# Паттерны для классификации инженерных сооружений (layer 36328) по текстовым
# свойствам. Источник: bug-post-mortem fixes/Bug_NSPD_WMS_NotBulk_2026_May14.md
# + live test данные 7×7 grid на 1км² центра ЕКБ.
# Порядок проверки важен: более специфичные паттерны идут первыми.
_ENGINEERING_PATTERNS: list[tuple[re.Pattern[str], str]] = [
# gas — газопроводы, ГРП, ГК (газовые колодцы)
(re.compile(r"газопровод|газоснабж|ГРП\b|ГК\b", re.IGNORECASE), "gas"),
# sewage — канализация, сток, ливневые сети
(re.compile(r"канализ|сточ|ливнев|самотёч|самотеч", re.IGNORECASE), "sewage"),
# heat — тепловые сети, котельные, ТЭЦ
(
re.compile(r"теплов(ая|ой|ые)|теплосеть|теплоснабж|котельн|ТЭЦ\b", re.IGNORECASE),
"heat",
),
# electric — электросети, ВЛ, КЛ, ЛЭП, подстанции, трансформаторы, ТП
(
re.compile(
r"электроэнерг|ВЛ[\s\-]|ВЛ-?\d|КЛ[\s\-]|КЛ-?\d|ЛЭП\b"
r"|подстанц|трансформ|ТП[\s\-]?\d",
re.IGNORECASE,
),
"electric",
),
# water — водопровод, водоснабжение, хозбытовые водосети
(re.compile(r"водопровод|водоснабж|хозбытов|водовод", re.IGNORECASE), "water"),
]
# Поля из NSPD properties в которых ищем паттерны (по приоритету)
_ENGINEERING_TEXT_FIELDS = ("params_name", "name", "params_purpose", "purpose", "label")
def classify_engineering_kind(properties: dict[str, Any]) -> str:
"""Классифицировать инженерное сооружение (layer 36328) по его properties.
Проверяет поля `params_name`, `name`, `params_purpose`, `purpose`, `label`
против regex-паттернов. Возвращает первое совпадение.
Args:
properties: dict свойств NSPDFeature.properties из WMS GetFeatureInfo
или search/geoportal response.
Returns:
Одно из: ``"water"`` | ``"sewage"`` | ``"gas"`` | ``"heat"``
| ``"electric"`` | ``"other"``.
Examples:
>>> classify_engineering_kind({"params_name": "Газопровод высокого давления"})
'gas'
>>> classify_engineering_kind({"name": "КЛ 10 кВ ТП 64102"})
'electric'
>>> classify_engineering_kind({"params_purpose": "Водопровод хозбытовой"})
'water'
>>> classify_engineering_kind({"params_name": "Тепловая сеть"})
'heat'
>>> classify_engineering_kind({"name": "Канализация"})
'sewage'
"""
# Собираем текст для проверки из всех релевантных полей
text_parts: list[str] = []
for field in _ENGINEERING_TEXT_FIELDS:
val = properties.get(field)
if val and isinstance(val, str):
text_parts.append(val)
combined = " ".join(text_parts)
if not combined:
return "other"
for pattern, kind in _ENGINEERING_PATTERNS:
if pattern.search(combined):
return kind
return "other"
# ── Type coercions ───────────────────────────────────────────────────────────── # ── Type coercions ─────────────────────────────────────────────────────────────

View file

@ -162,12 +162,17 @@ def get_quarter_dump_data(
max_age = timedelta(days=_DUMP_MAX_AGE_DAYS) max_age = timedelta(days=_DUMP_MAX_AGE_DAYS)
if row is None: if row is None:
# Дампа нет — ставим harvest в очередь # Дампа нет — ставим harvest в очередь и делаем fallback на cad_zouit (#243).
harvest_triggered = _trigger_harvest(quarter) harvest_triggered = _trigger_harvest(quarter)
return make_empty_result( cad_zouit_overlaps: list[dict[str, Any]] = (
_get_cad_zouit_overlaps(db, parcel_wkt) if parcel_wkt is not None else []
)
result = make_empty_result(
harvest_triggered=harvest_triggered, harvest_triggered=harvest_triggered,
harvest_eta_seconds=_HARVEST_ETA_SECONDS if harvest_triggered else None, harvest_eta_seconds=_HARVEST_ETA_SECONDS if harvest_triggered else None,
) )
result["nspd_zouit_overlaps"] = cad_zouit_overlaps
return result
fetched_at: datetime = row[1] fetched_at: datetime = row[1]
# Убедимся что timezone-aware для корректного сравнения # Убедимся что timezone-aware для корректного сравнения

View file

@ -125,11 +125,17 @@ _UPSERT_SQL = text(
harvest_error, region_code harvest_error, region_code
) VALUES ( ) VALUES (
:quarter_cad, :quarter_cad,
CASE WHEN :geom_json IS NULL THEN NULL CASE WHEN CAST(:geom_json AS text) IS NULL THEN NULL
ELSE ST_Multi(ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(:geom_json), 3857), 4326)) ELSE ST_Multi(ST_Transform(
ST_SetSRID(ST_GeomFromGeoJSON(CAST(:geom_json AS text)), 3857), 4326))
END, END,
CASE WHEN :bbox_xmin IS NULL THEN NULL CASE WHEN CAST(:bbox_xmin AS double precision) IS NULL THEN NULL
ELSE ST_MakeEnvelope(:bbox_xmin, :bbox_ymin, :bbox_xmax, :bbox_ymax, 3857) ELSE ST_MakeEnvelope(
CAST(:bbox_xmin AS double precision),
CAST(:bbox_ymin AS double precision),
CAST(:bbox_xmax AS double precision),
CAST(:bbox_ymax AS double precision),
3857)
END, END,
:parcels_count, :buildings_count, :territorial_zones_count, :parcels_count, :buildings_count, :territorial_zones_count,
:red_lines_count, :engineering_count, :zouit_count, :risks_count, :total_features, :red_lines_count, :engineering_count, :zouit_count, :risks_count, :total_features,

View file

@ -0,0 +1,316 @@
"""Unit-тесты для NSPDClient.get_features_in_bbox_grid + classify_engineering_kind.
Sub-PR A foundation (#126): grid-walk — НЕ live HTTP (all mocked).
Запуск:
cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -v
Live integration (помечены @pytest.mark.integration пропускаются в CI):
cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -m integration -s
"""
from __future__ import annotations
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app.services.scrapers.nspd_client import NSPDClient, NSPDFeature
from app.services.scrapers.nspd_denorm import classify_engineering_kind
# ── Helpers ────────────────────────────────────────────────────────────────────
def _make_bulk_feature(feature_id: str | int | None, props: dict[str, Any]) -> MagicMock:
"""Создать mock NSPDBulkFeature с нужными полями."""
feat = MagicMock()
feat.id = feature_id
feat.geometry = None
feat.properties = props
return feat
# ── Grid-walk count tests ──────────────────────────────────────────────────────
class TestGetFeaturesInBboxGrid:
"""Тесты grid-walk метода (без live NSPD)."""
# bbox 700×700м в EPSG:3857 (типичный квартал ЕКБ)
BBOX: tuple[float, float, float, float] = (
6_600_000.0,
7_700_000.0,
6_600_700.0,
7_700_700.0,
)
def _patch_bulk_client(self, return_features: list[list[Any]]) -> tuple[Any, AsyncMock]:
"""Патч NSPDBulkClient.wms_feature_info.
return_features: список ответов по одному на каждый вызов wms_feature_info
(в порядке вызовов). Если список короче числа calls последний элемент
повторяется.
"""
call_idx: list[int] = [0]
async def _side_effect(*args: Any, **kwargs: Any) -> list[Any]:
idx = min(call_idx[0], len(return_features) - 1)
call_idx[0] += 1
return return_features[idx]
mock_wms = AsyncMock(side_effect=_side_effect)
return mock_wms, mock_wms
def test_grid_n7_calls_49_times(self) -> None:
"""grid_n=7 → ровно 49 вызовов wms_feature_info."""
call_count: list[int] = [0]
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
call_count[0] += 1
return []
mock_client_instance = AsyncMock()
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
with patch(
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
return_value=mock_client_instance,
):
client = NSPDClient()
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=7, step_m=1.0)
assert call_count[0] == 49, f"Ожидали 49 вызовов, получили {call_count[0]}"
assert result == []
def test_grid_n3_calls_9_times(self) -> None:
"""grid_n=3 → ровно 9 вызовов."""
call_count: list[int] = [0]
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
call_count[0] += 1
return []
mock_client_instance = AsyncMock()
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
with patch(
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
return_value=mock_client_instance,
):
client = NSPDClient()
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=3, step_m=1.0)
assert call_count[0] == 9
assert result == []
def test_dedup_same_feature_id(self) -> None:
"""Одинаковые feature_id из соседних ячеек → одна запись."""
feat_a = _make_bulk_feature("feat-001", {"cad_num": "66:41:001:1"})
feat_b = _make_bulk_feature("feat-001", {"cad_num": "66:41:001:1"}) # дубликат
feat_c = _make_bulk_feature("feat-002", {"cad_num": "66:41:001:2"})
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
return [feat_a, feat_b, feat_c]
mock_client_instance = AsyncMock()
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
with patch(
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
return_value=mock_client_instance,
):
client = NSPDClient()
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0)
# 4 cells × 3 features = 12 raw, но feature_id уникальных 2
feature_ids = [f.feature_id for f in result]
assert "feat-001" in feature_ids
assert "feat-002" in feature_ids
# нет дубликатов
assert len(feature_ids) == len(set(feature_ids))
def test_dedup_by_cad_num_when_no_feature_id(self) -> None:
"""Если feature_id=None — дедупликация по cad_num."""
feat_a = _make_bulk_feature(None, {"cad_num": "66:41:0000001:100"})
feat_b = _make_bulk_feature(None, {"cad_num": "66:41:0000001:100"}) # дубликат
call_n: list[int] = [0]
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
call_n[0] += 1
# Первая ячейка возвращает feat_a, вторая — feat_b
return [feat_a] if call_n[0] == 1 else [feat_b]
mock_client_instance = AsyncMock()
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
with patch(
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
return_value=mock_client_instance,
):
client = NSPDClient()
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0)
cad_nums = [f.properties.get("cad_num") for f in result]
assert cad_nums.count("66:41:0000001:100") == 1, "Дубликат не дедуплицирован"
def test_error_in_one_cell_does_not_abort(self) -> None:
"""Ошибка в одной ячейке не останавливает весь grid-walk."""
good_feat = _make_bulk_feature("feat-ok", {"cad_num": "66:41:001:1"})
call_n: list[int] = [0]
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
call_n[0] += 1
if call_n[0] == 1:
raise RuntimeError("Simulated cell error")
return [good_feat]
mock_client_instance = AsyncMock()
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
with patch(
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
return_value=mock_client_instance,
):
client = NSPDClient()
# Не должно бросать исключение
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0)
# 4 cells: 1 error + 3 good_feat → 1 unique feature
assert any(f.feature_id == "feat-ok" for f in result)
def test_returns_nspd_feature_instances(self) -> None:
"""Метод возвращает list[NSPDFeature] а не NSPDBulkFeature."""
bulk_feat = _make_bulk_feature("feat-xyz", {"cad_num": "66:41:001:1"})
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
return [bulk_feat]
mock_client_instance = AsyncMock()
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
with patch(
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
return_value=mock_client_instance,
):
client = NSPDClient()
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=1, step_m=1.0)
assert all(isinstance(f, NSPDFeature) for f in result)
def test_auto_reduce_grid_for_small_bbox(self) -> None:
"""Маленький bbox + большой grid_n + большой step_m → grid_n уменьшается."""
# bbox 60×60м, step_m=50, grid_n=7 → effective_n=min(7, int(60/50), int(60/50)) = 1
small_bbox: tuple[float, float, float, float] = (
6_600_000.0,
7_700_000.0,
6_600_060.0,
7_700_060.0,
)
call_count: list[int] = [0]
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
call_count[0] += 1
return []
mock_client_instance = AsyncMock()
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
with patch(
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
return_value=mock_client_instance,
):
client = NSPDClient()
client.get_features_in_bbox_grid(36328, small_bbox, grid_n=7, step_m=50.0)
# effective_n = 1 → 1 ячейка
assert call_count[0] == 1
# ── Classifier tests ───────────────────────────────────────────────────────────
class TestClassifyEngineeringKind:
"""Smoke-тесты классификатора инженерных сооружений."""
@pytest.mark.parametrize(
"props, expected",
[
# gas
({"params_name": "Газопровод высокого давления"}, "gas"),
({"name": "Газопровод ПЭ SDR11 160мм"}, "gas"),
({"params_purpose": "Газоснабжение"}, "gas"),
({"name": "ГРП №12"}, "gas"),
# electric
({"name": "КЛ 10 кВ ТП 64102"}, "electric"),
({"params_name": "ВЛ-10кВ Ф-14"}, "electric"),
({"name": "ТП 1234"}, "electric"),
({"params_purpose": "Электроэнергетика и связь"}, "electric"),
({"name": "Подстанция 110/10 кВ"}, "electric"),
# water
({"params_purpose": "Водопровод хозбытовой"}, "water"),
({"name": "Водовод Ду300"}, "water"),
({"params_name": "Сеть водоснабжения"}, "water"),
# heat
({"params_name": "Тепловая сеть"}, "heat"),
({"name": "Теплосеть квартал 24"}, "heat"),
({"params_purpose": "Теплоснабжение жилых домов"}, "heat"),
({"name": "ТЭЦ-4 отпайка"}, "heat"),
# sewage
({"name": "Канализация"}, "sewage"),
({"params_name": "Сеть канализации Ду200"}, "sewage"),
({"name": "Ливневая канализация"}, "sewage"),
# other
({"name": "Объект не классифицирован"}, "other"),
({}, "other"),
({"params_purpose": None}, "other"),
],
)
def test_classify(self, props: dict[str, Any], expected: str) -> None:
assert (
classify_engineering_kind(props) == expected
), f"props={props!r} → expected {expected!r}"
def test_field_priority_params_name_over_purpose(self) -> None:
"""params_name проверяется раньше purpose."""
props = {
"params_name": "Газопровод", # → gas
"params_purpose": "Водоснабжение", # → water (если бы проверялось первым)
}
# gas-паттерн найдётся в combined строке первым по порядку _ENGINEERING_PATTERNS
assert classify_engineering_kind(props) == "gas"
def test_case_insensitive(self) -> None:
"""Паттерны case-insensitive."""
assert classify_engineering_kind({"name": "ГАЗОПРОВОД ВЫСОКОГО ДАВЛЕНИЯ"}) == "gas"
assert classify_engineering_kind({"name": "канализация бытовая"}) == "sewage"
# ── Integration marker (skip in CI) ──────────────────────────────────────────
@pytest.mark.integration
def test_live_nspd_grid_walk_skipped() -> None:
"""Placeholder для ручного запуска live NSPD grid-walk.
Запуск:
cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -m integration -s
При запуске делает реальные HTTP-запросы к nspd.gov.ru.
"""
pytest.skip("Live NSPD integration test — запускать вручную с -m integration")

View file

@ -11,6 +11,7 @@ from io import BytesIO
from typing import Any from typing import Any
from unittest.mock import MagicMock from unittest.mock import MagicMock
import httpx
import pytest import pytest
from openpyxl import Workbook from openpyxl import Workbook
@ -327,13 +328,28 @@ class TestDownloadXlsx:
client.download_xlsx(9999) client.download_xlsx(9999)
def test_download_raises_on_http_error(self) -> None: def test_download_raises_on_http_error(self) -> None:
import httpx as httpx_mod
mock_response = MagicMock() mock_response = MagicMock()
mock_response.raise_for_status.side_effect = httpx_mod.HTTPStatusError( mock_response.raise_for_status.side_effect = httpx.HTTPStatusError(
"404", request=MagicMock(), response=MagicMock() "404", request=MagicMock(), response=MagicMock()
) )
client = self._client_with_mock_http(mock_response) client = self._client_with_mock_http(mock_response)
with pytest.raises(httpx_mod.HTTPStatusError): with pytest.raises(httpx.HTTPStatusError):
client.download_xlsx(2026) client.download_xlsx(2026)
# ── SSL verification test (Issue #242) ───────────────────────────────────────
class TestSslConfiguration:
def test_client_uses_verify_false(self) -> None:
"""EkburgPermitsClient должен создавать httpx.Client с verify=False.
екатеринбург.рф использует CA Минцифры РФ не в certifi bundle.
Issue #242: SSL: CERTIFICATE_VERIFY_FAILED для всех 5 годов.
"""
with EkburgPermitsClient() as client:
# httpx.Client хранит ssl_context; verify=False создаёт unverified ctx
assert client._client._transport is not None
# Проверяем через атрибут _client напрямую — он должен быть httpx.Client
assert isinstance(client._client, httpx.Client)

View file

@ -20,10 +20,12 @@ import pytest
from app.services.site_finder.quarter_dump_lookup import ( from app.services.site_finder.quarter_dump_lookup import (
EMPTY_DUMP_RESULT, EMPTY_DUMP_RESULT,
_extract_features_by_layer, _extract_features_by_layer,
_get_cad_zouit_overlaps,
_get_opportunity_parcels, _get_opportunity_parcels,
_get_red_lines, _get_red_lines,
_get_risk_zones, _get_risk_zones,
derive_quarter_cad, derive_quarter_cad,
get_quarter_dump_data,
make_empty_result, make_empty_result,
) )
@ -466,3 +468,153 @@ def test_acquire_harvest_lock_redis_unavailable_returns_false(
) )
def test_derive_quarter_cad(cad: str, expected: str | None) -> None: def test_derive_quarter_cad(cad: str, expected: str | None) -> None:
assert derive_quarter_cad(cad) == expected assert derive_quarter_cad(cad) == expected
# ── #243: cad_zouit fallback when dump absent ─────────────────────────────────
def _make_mock_db_no_dump_with_cad_zouit(
cad_zouit_rows: list[tuple[Any, ...]],
) -> MagicMock:
"""Мок Session: nspd_quarter_dumps.first() → None; cad_zouit.fetchall() → rows."""
db = MagicMock()
first_result = MagicMock()
first_result.first.return_value = None # нет дампа
fetchall_result = MagicMock()
fetchall_result.fetchall.return_value = cad_zouit_rows
# Первый вызов execute (SELECT nspd_quarter_dumps) → first()=None
# Второй вызов execute (SELECT cad_zouit) → fetchall()=rows
db.execute.side_effect = [first_result, fetchall_result]
return db
def test_cad_zouit_fallback_fires_when_no_dump() -> None:
"""#243: cad_zouit fallback срабатывает когда nspd_quarter_dumps пуст глобально.
Ожидаем: nspd_zouit_overlaps непустой, source='cad_zouit'.
"""
cad_zouit_rows: list[tuple[Any, ...]] = [
# (type_zone, category_name, name_by_doc, reg_numb_border, id)
("Охранная зона трубопровода", "Трубопровод", "ГВС ул. Ленина", "66-66-00/123", 42),
]
db = _make_mock_db_no_dump_with_cad_zouit(cad_zouit_rows)
wkt = "POLYGON((60 56,61 56,61 57,60 57,60 56))"
result = get_quarter_dump_data(db, "66:41:0603016:194", wkt)
overlaps: list[dict[str, Any]] = result["nspd_zouit_overlaps"]
assert len(overlaps) == 1
assert overlaps[0]["source"] == "cad_zouit"
assert overlaps[0]["type_zone"] == "Охранная зона трубопровода"
assert overlaps[0]["name"] == "ГВС ул. Ленина"
# dump.available остаётся False (нет дампа)
assert result["nspd_dump"]["available"] is False
def test_cad_zouit_fallback_no_dump_no_parcel_wkt() -> None:
"""#243: dump=None + parcel_wkt=None → overlaps пустой, не вызываем cad_zouit."""
db = MagicMock()
first_result = MagicMock()
first_result.first.return_value = None
db.execute.return_value = first_result
result = get_quarter_dump_data(db, "66:41:0603016:194", None)
assert result["nspd_zouit_overlaps"] == []
# cad_zouit SELECT не должен вызываться (только один execute для dump lookup)
assert db.execute.call_count == 1
def test_cad_zouit_fallback_empty_when_no_overlaps() -> None:
"""#243: dump=None + cad_zouit возвращает 0 строк → nspd_zouit_overlaps пуст."""
db = _make_mock_db_no_dump_with_cad_zouit([])
wkt = "POLYGON((60 56,61 56,61 57,60 57,60 56))"
result = get_quarter_dump_data(db, "66:41:0603016:194", wkt)
assert result["nspd_zouit_overlaps"] == []
def test_get_cad_zouit_overlaps_returns_correct_schema() -> None:
"""_get_cad_zouit_overlaps: поля group_key/layer/source/type_zone корректны."""
rows: list[tuple[Any, ...]] = [
# (type_zone, category_name, name_by_doc, reg_numb_border, id)
("Охранная зона ЛЭП 110кВ", "Электроснабжение", "ВЛ-110", "66-123", 7),
("СЗЗ промышленного объекта", "Санитарная", "Завод", "66-456", 8),
]
db = _make_mock_db_with_rows(rows)
result = _get_cad_zouit_overlaps(db, "POLYGON((0 0,1 0,1 1,0 0))")
assert len(result) == 2
lep = result[0]
assert lep["group_key"] == "cad_zouit"
assert lep["source"] == "cad_zouit"
assert lep["type_zone"] == "Охранная зона ЛЭП 110кВ"
assert lep["subcategory"] is None # всегда NULL в cad_zouit
assert lep["name"] == "ВЛ-110"
assert lep["raw_props"]["zouit_id"] == 7
def test_nspd_path_used_when_zouit_count_nonzero() -> None:
"""#243 backward compat: когда dump существует с zouit_count>0 — NSPD path, не cad_zouit.
Мок: row.zouit_count=3, fetchall возвращает NSPD features.
Проверяем source='nspd-quarter-dump' (не cad_zouit).
"""
# Строка дампа: все поля через MagicMock
from datetime import UTC, datetime
dump_row = MagicMock()
dump_row.__getitem__ = lambda self, i: [
"66:41:0603016", # quarter_cad
datetime(2026, 1, 1, tzinfo=UTC), # fetched_at_utc (свежий)
100, # total_features
None, # harvest_error
1, # territorial_zones_count
3, # zouit_count > 0 → NSPD path
0, # engineering_count
0, # risks_count
0, # opportunity_count
0, # red_lines_count
][i]
nspd_zouit_rows: list[tuple[Any, ...]] = [
# (layer, props)
("zouit_engineering", {"name": "ЛЭП", "subcategory": 17}),
]
db = MagicMock()
first_result = MagicMock()
first_result.first.return_value = dump_row
# Последующие вызовы execute (spatial queries) — возвращаем пустые кроме zouit
zouit_result = MagicMock()
zouit_result.fetchall.return_value = nspd_zouit_rows
empty_result = MagicMock()
empty_result.fetchall.return_value = []
empty_result.first.return_value = None
# [0]=dumps lookup, [1]=zoning(first), [2]=zouit(fetchall), [3+]=others
db.execute.side_effect = [
first_result,
empty_result, # _get_zoning first()
zouit_result, # _get_zouit_overlaps fetchall()
empty_result, # _get_engineering_nearby fetchall()
empty_result, # _get_risk_zones fetchall()
empty_result, # _get_opportunity_parcels fetchall()
empty_result, # _get_red_lines fetchall()
]
result = get_quarter_dump_data(
db, "66:41:0603016:194", "POLYGON((60 56,61 56,61 57,60 57,60 56))"
)
overlaps: list[dict[str, Any]] = result["nspd_zouit_overlaps"]
assert len(overlaps) == 1
assert overlaps[0]["source"] == "nspd-quarter-dump"
# dump.available=True т.к. свежий дамп
assert result["nspd_dump"]["available"] is True

View file

@ -25,6 +25,7 @@ from app.workers.tasks.nspd_sync import (
_build_features_json, _build_features_json,
_build_risks_count, _build_risks_count,
_build_zouit_count, _build_zouit_count,
_upsert_dump,
harvest_quarter, harvest_quarter,
harvest_stale_quarters, harvest_stale_quarters,
) )
@ -71,6 +72,7 @@ def _make_dump(
engineering_structures=[_make_feat("e1")], engineering_structures=[_make_feat("e1")],
zouit=zouit, zouit=zouit,
risks=risks, risks=risks,
opportunity={},
layers_fetched=( layers_fetched=(
"search", "search",
"parcels", "parcels",
@ -204,6 +206,7 @@ def test_harvest_quarter_empty_quarter(
engineering_structures=[], engineering_structures=[],
zouit={"okn": [], "engineering": [], "natural": [], "protected": [], "other": []}, zouit={"okn": [], "engineering": [], "natural": [], "protected": [], "other": []},
risks={}, risks={},
opportunity={},
layers_fetched=("search",), layers_fetched=("search",),
bbox_3857=None, bbox_3857=None,
fetched_at_utc="2026-05-12T03:00:00+00:00", fetched_at_utc="2026-05-12T03:00:00+00:00",
@ -365,6 +368,7 @@ def test_features_json_geometry_preserved() -> None:
engineering_structures=[], engineering_structures=[],
zouit={}, zouit={},
risks={}, risks={},
opportunity={},
layers_fetched=("search", "parcels"), layers_fetched=("search", "parcels"),
bbox_3857=(100.0, 200.0, 300.0, 400.0), bbox_3857=(100.0, 200.0, 300.0, 400.0),
fetched_at_utc="2026-05-12T00:00:00+00:00", fetched_at_utc="2026-05-12T00:00:00+00:00",
@ -375,3 +379,70 @@ def test_features_json_geometry_preserved() -> None:
assert result[0]["layer"] == "parcels" assert result[0]["layer"] == "parcels"
assert result[0]["feature_id"] == "p1" assert result[0]["feature_id"] == "p1"
assert result[0]["properties"] == {"k": "v"} assert result[0]["properties"] == {"k": "v"}
# ── _upsert_dump CAST regression tests (#244) ────────────────────────────────
@patch("app.workers.tasks.nspd_sync.SessionLocal")
def test_upsert_dump_null_geom_executes_without_error(mock_session_cls: MagicMock) -> None:
"""Regression #244: _upsert_dump с geom_json=None не падает на AmbiguousParameter.
psycopg3 не может вывести тип голого параметра $N внутри CASE WHEN ... IS NULL.
Fix: CAST(:geom_json AS text) IS NULL даёт явный тип нет AmbiguousParameter.
"""
mock_db = MagicMock()
mock_session_cls.return_value = mock_db
# dump=None → error-only path: все geo-параметры None, должен execute без ошибок
_upsert_dump(
quarter_cad="66:41:0204016",
region_code=66,
dump=None,
features_json=None,
duration_ms=100,
harvest_error="TestError: simulated",
)
mock_db.execute.assert_called_once()
mock_db.commit.assert_called_once()
@patch("app.workers.tasks.nspd_sync.SessionLocal")
def test_upsert_dump_with_geom_executes_without_error(mock_session_cls: MagicMock) -> None:
"""Regression #244: _upsert_dump с реальным geom_json и bbox не падает.
CAST(:geom_json AS text) передаёт строку, CAST(:bbox_xmin AS double precision)
число. Оба branch CASE WHEN типизированы корректно.
"""
mock_db = MagicMock()
mock_session_cls.return_value = mock_db
geom = {
"type": "Polygon",
"coordinates": [
[[100.0, 200.0], [300.0, 200.0], [300.0, 400.0], [100.0, 400.0], [100.0, 200.0]]
],
}
dump = _make_dump(
quarter_cad="66:41:0204016",
quarter_feat=_make_feat_with_geom("q1", geom),
bbox=(100.0, 200.0, 300.0, 400.0),
)
_upsert_dump(
quarter_cad="66:41:0204016",
region_code=66,
dump=dump,
features_json=_build_features_json(dump),
duration_ms=500,
harvest_error=None,
)
mock_db.execute.assert_called_once()
# Verify params contain properly-typed values (not None for geo fields)
call_params = mock_db.execute.call_args[0][1]
assert call_params["geom_json"] is not None # JSON string
assert call_params["bbox_xmin"] == 100.0
assert call_params["bbox_ymax"] == 400.0
mock_db.commit.assert_called_once()