Merge remote-tracking branch 'forgejo/main' into fix/234-nspd-harvest-eta-badge
# Conflicts: # backend/app/services/site_finder/quarter_dump_lookup.py
This commit is contained in:
commit
b99ac76463
9 changed files with 811 additions and 21 deletions
|
|
@ -248,10 +248,13 @@ class EkburgPermitsClient:
|
||||||
USER_AGENT = "GenDesign/1.0 (+https://gendsgn.ru) Site Finder permits scraper"
|
USER_AGENT = "GenDesign/1.0 (+https://gendsgn.ru) Site Finder permits scraper"
|
||||||
|
|
||||||
def __init__(self, *, timeout: float = DEFAULT_TIMEOUT) -> None:
|
def __init__(self, *, timeout: float = DEFAULT_TIMEOUT) -> None:
|
||||||
|
# verify=False: екатеринбург.рф подписан CA Минцифры РФ (нет в certifi).
|
||||||
|
# Данные публичные open-data — SSL pinning здесь не требуется. Issue #242.
|
||||||
self._client = httpx.Client(
|
self._client = httpx.Client(
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
follow_redirects=True,
|
follow_redirects=True,
|
||||||
headers={"User-Agent": self.USER_AGENT},
|
headers={"User-Agent": self.USER_AGENT},
|
||||||
|
verify=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
def __enter__(self) -> EkburgPermitsClient:
|
def __enter__(self) -> EkburgPermitsClient:
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,7 @@ WMS endpoints (per #94 issue body, TIER 1-6 каталог слоёв):
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import datetime as _dt
|
import datetime as _dt
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
|
@ -384,24 +385,22 @@ class NSPDClient:
|
||||||
width: int = 4096,
|
width: int = 4096,
|
||||||
height: int = 4096,
|
height: int = 4096,
|
||||||
) -> list[NSPDFeature]:
|
) -> list[NSPDFeature]:
|
||||||
"""Bulk fetch features в bbox через GetFeatureInfo с большим bbox.
|
"""WMS GetFeatureInfo на одном центральном пикселе bbox.
|
||||||
|
|
||||||
Workaround: WFS GetCapabilities → 404 на nspd.gov.ru, нет WFS
|
DEPRECATED: возвращает 0-3 features под одним пикселем (I=W/2, J=H/2).
|
||||||
GetFeature endpoint. Решение: использовать GetFeatureInfo с large
|
НЕ является bulk fetch несмотря на исходный docstring — WMS GetFeatureInfo
|
||||||
bbox и точкой в центре (I=W/2, J=H/2) — возвращает все features
|
по стандарту OGC возвращает объекты строго под одной pixel-точкой, а не
|
||||||
пересекающиеся с bbox.
|
во всём bbox. Для получения всех объектов в bbox используй
|
||||||
|
`get_features_in_bbox_grid`.
|
||||||
|
|
||||||
|
See: fixes/Bug_NSPD_WMS_NotBulk_2026_May14.md
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
bbox_3857: (xmin, ymin, xmax, ymax) в EPSG:3857 метрах.
|
bbox_3857: (xmin, ymin, xmax, ymax) в EPSG:3857 метрах.
|
||||||
width/height: размер виртуального tile. Большой → большой bbox.
|
width/height: размер виртуального tile.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
list[NSPDFeature]; пусто если ничего не найдено.
|
list[NSPDFeature]; пусто если ничего не найдено.
|
||||||
|
|
||||||
Use cases (per #94 acceptance):
|
|
||||||
- sync_territorial_zones_bbox → закрывает G1 #28 ПЗЗ
|
|
||||||
- sync_zouit_*_bbox → G3 #30
|
|
||||||
- sync_risk_zones_bbox → новый risk overlay
|
|
||||||
"""
|
"""
|
||||||
xmin, ymin, xmax, ymax = bbox_3857
|
xmin, ymin, xmax, ymax = bbox_3857
|
||||||
params = {
|
params = {
|
||||||
|
|
@ -426,6 +425,149 @@ class NSPDClient:
|
||||||
feats = (data or {}).get("features") or []
|
feats = (data or {}).get("features") or []
|
||||||
return [NSPDFeature.from_raw(f) for f in feats]
|
return [NSPDFeature.from_raw(f) for f in feats]
|
||||||
|
|
||||||
|
# ── 3b. get_features_in_bbox_grid ───────────────────────────────────────
|
||||||
|
|
||||||
|
def get_features_in_bbox_grid(
|
||||||
|
self,
|
||||||
|
layer_id: int,
|
||||||
|
bbox: tuple[float, float, float, float],
|
||||||
|
*,
|
||||||
|
grid_n: int = 7,
|
||||||
|
step_m: float = 50.0,
|
||||||
|
tile_size: int = 512,
|
||||||
|
) -> list[NSPDFeature]:
|
||||||
|
"""Bulk-аппроксимация bbox через grid-walk WMS GetFeatureInfo.
|
||||||
|
|
||||||
|
Разбивает bbox на grid_n × grid_n равных ячеек. В каждой ячейке
|
||||||
|
вызывает WMS GetFeatureInfo в центральном пикселе. Дедуплицирует
|
||||||
|
результаты по feature_id / cad_num / reg_numb_border — возвращает
|
||||||
|
список уникальных NSPDFeature.
|
||||||
|
|
||||||
|
Делегирует HTTP через NSPDBulkClient.wms_feature_info (async httpx
|
||||||
|
с semaphore и retry), запуская asyncio event loop синхронно через
|
||||||
|
asyncio.run(). Предназначен для вызова из синхронного кода (Celery
|
||||||
|
task, FastAPI sync handler).
|
||||||
|
|
||||||
|
Concurrency: NSPDBulkClient._SEMAPHORE(3) ограничивает параллельные
|
||||||
|
запросы. При grid_n=7 (49 ячеек) — все 49 ячеек запускаются одним
|
||||||
|
gather; семафор пропускает не более 3 одновременно. Thread-safety:
|
||||||
|
каждый вызов get_features_in_bbox_grid создаёт новый event loop
|
||||||
|
через asyncio.run() — безопасно из разных Celery workers (process-
|
||||||
|
уровень изоляции).
|
||||||
|
|
||||||
|
Args:
|
||||||
|
layer_id: NSPD layer ID (например 36328 сооружения, 37578 ЗОУИТ).
|
||||||
|
bbox: (xmin, ymin, xmax, ymax) в EPSG:3857 (метры).
|
||||||
|
grid_n: размер сетки по каждой оси. 7 → 49 запросов (~coarse),
|
||||||
|
15 → 225 запросов (~fine). По умолчанию 7 для первичного scan.
|
||||||
|
step_m: минимальный шаг ячейки в метрах. Если bbox меньше
|
||||||
|
grid_n*step_m — grid_n уменьшается автоматически чтобы
|
||||||
|
ячейки не становились меньше step_m.
|
||||||
|
tile_size: размер виртуального WMS тайла (пиксели).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Дедуплицированный list[NSPDFeature]. Может быть пуст если в bbox
|
||||||
|
нет объектов данного layer'а.
|
||||||
|
|
||||||
|
Note:
|
||||||
|
Не делает live HTTP вызовы если вызван с mock NSPDBulkClient.
|
||||||
|
Rate-limit управляется семафором NSPDBulkClient._SEMAPHORE(3) +
|
||||||
|
asyncio.sleep(0.05) jitter — не через self.rate_ms.
|
||||||
|
"""
|
||||||
|
# Импортируем здесь чтобы избежать circular import:
|
||||||
|
# nspd_client ← nspd_bulk_client (оба top-level scrapers, не cross-domain)
|
||||||
|
from app.scrapers.nspd_bulk_client import NSPDBulkClient
|
||||||
|
|
||||||
|
xmin, ymin, xmax, ymax = bbox
|
||||||
|
width_m = xmax - xmin
|
||||||
|
height_m = ymax - ymin
|
||||||
|
|
||||||
|
# Авто-коррекция grid_n если bbox мал для шага step_m
|
||||||
|
effective_n = min(
|
||||||
|
grid_n,
|
||||||
|
max(1, int(width_m / step_m)),
|
||||||
|
max(1, int(height_m / step_m)),
|
||||||
|
)
|
||||||
|
if effective_n < grid_n:
|
||||||
|
logger.info(
|
||||||
|
"get_features_in_bbox_grid layer=%d: bbox %.0fx%.0fm < grid_n=%d×step_m=%.0f"
|
||||||
|
" — уменьшаем grid до %d×%d",
|
||||||
|
layer_id,
|
||||||
|
width_m,
|
||||||
|
height_m,
|
||||||
|
grid_n,
|
||||||
|
step_m,
|
||||||
|
effective_n,
|
||||||
|
effective_n,
|
||||||
|
)
|
||||||
|
|
||||||
|
x_step = width_m / effective_n
|
||||||
|
y_step = height_m / effective_n
|
||||||
|
|
||||||
|
# Генерируем список (sub_bbox, click_xy) ячеек
|
||||||
|
cells: list[tuple[tuple[float, float, float, float], tuple[int, int]]] = []
|
||||||
|
click_px = tile_size // 2
|
||||||
|
for i in range(effective_n):
|
||||||
|
for j in range(effective_n):
|
||||||
|
cell_xmin = xmin + i * x_step
|
||||||
|
cell_ymin = ymin + j * y_step
|
||||||
|
cell_xmax = cell_xmin + x_step
|
||||||
|
cell_ymax = cell_ymin + y_step
|
||||||
|
cells.append(((cell_xmin, cell_ymin, cell_xmax, cell_ymax), (click_px, click_px)))
|
||||||
|
|
||||||
|
async def _run_grid() -> list[NSPDFeature]:
|
||||||
|
async with NSPDBulkClient() as client:
|
||||||
|
tasks = [
|
||||||
|
client.wms_feature_info(layer_id, sub_bbox, click_xy, tile_size, tile_size)
|
||||||
|
for sub_bbox, click_xy in cells
|
||||||
|
]
|
||||||
|
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||||
|
|
||||||
|
features: list[NSPDFeature] = []
|
||||||
|
for r in results:
|
||||||
|
if isinstance(r, Exception):
|
||||||
|
logger.warning("get_features_in_bbox_grid layer=%d cell error: %s", layer_id, r)
|
||||||
|
continue
|
||||||
|
for bulk_feat in r:
|
||||||
|
raw = {
|
||||||
|
"id": bulk_feat.id,
|
||||||
|
"geometry": bulk_feat.geometry,
|
||||||
|
"properties": bulk_feat.properties,
|
||||||
|
}
|
||||||
|
features.append(NSPDFeature.from_raw(raw))
|
||||||
|
return features
|
||||||
|
|
||||||
|
raw_features = asyncio.run(_run_grid())
|
||||||
|
|
||||||
|
# Дедупликация — приоритет ключей: feature_id > cad_num > reg_numb_border
|
||||||
|
seen: set[str] = set()
|
||||||
|
deduped: list[NSPDFeature] = []
|
||||||
|
for feat in raw_features:
|
||||||
|
props = feat.properties
|
||||||
|
dedup_key = (
|
||||||
|
feat.feature_id
|
||||||
|
or props.get("cad_num")
|
||||||
|
or props.get("cad_number")
|
||||||
|
or props.get("reg_numb_border")
|
||||||
|
or props.get("label")
|
||||||
|
)
|
||||||
|
if dedup_key is not None:
|
||||||
|
if dedup_key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(dedup_key)
|
||||||
|
deduped.append(feat)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"get_features_in_bbox_grid layer=%d grid=%dx%d cells=%d raw=%d deduped=%d",
|
||||||
|
layer_id,
|
||||||
|
effective_n,
|
||||||
|
effective_n,
|
||||||
|
len(cells),
|
||||||
|
len(raw_features),
|
||||||
|
len(deduped),
|
||||||
|
)
|
||||||
|
return deduped
|
||||||
|
|
||||||
# ── 4. list_layers ──────────────────────────────────────────────────────
|
# ── 4. list_layers ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
def list_layers(self, theme_id: int = THEME_PKK) -> list[NSPDLayer]:
|
def list_layers(self, theme_id: int = THEME_PKK) -> list[NSPDLayer]:
|
||||||
|
|
|
||||||
|
|
@ -32,6 +32,7 @@ from __future__ import annotations
|
||||||
import datetime
|
import datetime
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
import re
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
|
|
@ -39,6 +40,84 @@ from sqlalchemy.orm import Session
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# ── Engineering classifier ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
# Паттерны для классификации инженерных сооружений (layer 36328) по текстовым
|
||||||
|
# свойствам. Источник: bug-post-mortem fixes/Bug_NSPD_WMS_NotBulk_2026_May14.md
|
||||||
|
# + live test данные 7×7 grid на 1км² центра ЕКБ.
|
||||||
|
# Порядок проверки важен: более специфичные паттерны идут первыми.
|
||||||
|
|
||||||
|
_ENGINEERING_PATTERNS: list[tuple[re.Pattern[str], str]] = [
|
||||||
|
# gas — газопроводы, ГРП, ГК (газовые колодцы)
|
||||||
|
(re.compile(r"газопровод|газоснабж|ГРП\b|ГК\b", re.IGNORECASE), "gas"),
|
||||||
|
# sewage — канализация, сток, ливневые сети
|
||||||
|
(re.compile(r"канализ|сточ|ливнев|самотёч|самотеч", re.IGNORECASE), "sewage"),
|
||||||
|
# heat — тепловые сети, котельные, ТЭЦ
|
||||||
|
(
|
||||||
|
re.compile(r"теплов(ая|ой|ые)|теплосеть|теплоснабж|котельн|ТЭЦ\b", re.IGNORECASE),
|
||||||
|
"heat",
|
||||||
|
),
|
||||||
|
# electric — электросети, ВЛ, КЛ, ЛЭП, подстанции, трансформаторы, ТП
|
||||||
|
(
|
||||||
|
re.compile(
|
||||||
|
r"электроэнерг|ВЛ[\s\-]|ВЛ-?\d|КЛ[\s\-]|КЛ-?\d|ЛЭП\b"
|
||||||
|
r"|подстанц|трансформ|ТП[\s\-]?\d",
|
||||||
|
re.IGNORECASE,
|
||||||
|
),
|
||||||
|
"electric",
|
||||||
|
),
|
||||||
|
# water — водопровод, водоснабжение, хозбытовые водосети
|
||||||
|
(re.compile(r"водопровод|водоснабж|хозбытов|водовод", re.IGNORECASE), "water"),
|
||||||
|
]
|
||||||
|
|
||||||
|
# Поля из NSPD properties в которых ищем паттерны (по приоритету)
|
||||||
|
_ENGINEERING_TEXT_FIELDS = ("params_name", "name", "params_purpose", "purpose", "label")
|
||||||
|
|
||||||
|
|
||||||
|
def classify_engineering_kind(properties: dict[str, Any]) -> str:
|
||||||
|
"""Классифицировать инженерное сооружение (layer 36328) по его properties.
|
||||||
|
|
||||||
|
Проверяет поля `params_name`, `name`, `params_purpose`, `purpose`, `label`
|
||||||
|
против regex-паттернов. Возвращает первое совпадение.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
properties: dict свойств NSPDFeature.properties из WMS GetFeatureInfo
|
||||||
|
или search/geoportal response.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Одно из: ``"water"`` | ``"sewage"`` | ``"gas"`` | ``"heat"``
|
||||||
|
| ``"electric"`` | ``"other"``.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
>>> classify_engineering_kind({"params_name": "Газопровод высокого давления"})
|
||||||
|
'gas'
|
||||||
|
>>> classify_engineering_kind({"name": "КЛ 10 кВ ТП 64102"})
|
||||||
|
'electric'
|
||||||
|
>>> classify_engineering_kind({"params_purpose": "Водопровод хозбытовой"})
|
||||||
|
'water'
|
||||||
|
>>> classify_engineering_kind({"params_name": "Тепловая сеть"})
|
||||||
|
'heat'
|
||||||
|
>>> classify_engineering_kind({"name": "Канализация"})
|
||||||
|
'sewage'
|
||||||
|
"""
|
||||||
|
# Собираем текст для проверки из всех релевантных полей
|
||||||
|
text_parts: list[str] = []
|
||||||
|
for field in _ENGINEERING_TEXT_FIELDS:
|
||||||
|
val = properties.get(field)
|
||||||
|
if val and isinstance(val, str):
|
||||||
|
text_parts.append(val)
|
||||||
|
|
||||||
|
combined = " ".join(text_parts)
|
||||||
|
if not combined:
|
||||||
|
return "other"
|
||||||
|
|
||||||
|
for pattern, kind in _ENGINEERING_PATTERNS:
|
||||||
|
if pattern.search(combined):
|
||||||
|
return kind
|
||||||
|
|
||||||
|
return "other"
|
||||||
|
|
||||||
|
|
||||||
# ── Type coercions ─────────────────────────────────────────────────────────────
|
# ── Type coercions ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -162,12 +162,17 @@ def get_quarter_dump_data(
|
||||||
max_age = timedelta(days=_DUMP_MAX_AGE_DAYS)
|
max_age = timedelta(days=_DUMP_MAX_AGE_DAYS)
|
||||||
|
|
||||||
if row is None:
|
if row is None:
|
||||||
# Дампа нет — ставим harvest в очередь
|
# Дампа нет — ставим harvest в очередь и делаем fallback на cad_zouit (#243).
|
||||||
harvest_triggered = _trigger_harvest(quarter)
|
harvest_triggered = _trigger_harvest(quarter)
|
||||||
return make_empty_result(
|
cad_zouit_overlaps: list[dict[str, Any]] = (
|
||||||
|
_get_cad_zouit_overlaps(db, parcel_wkt) if parcel_wkt is not None else []
|
||||||
|
)
|
||||||
|
result = make_empty_result(
|
||||||
harvest_triggered=harvest_triggered,
|
harvest_triggered=harvest_triggered,
|
||||||
harvest_eta_seconds=_HARVEST_ETA_SECONDS if harvest_triggered else None,
|
harvest_eta_seconds=_HARVEST_ETA_SECONDS if harvest_triggered else None,
|
||||||
)
|
)
|
||||||
|
result["nspd_zouit_overlaps"] = cad_zouit_overlaps
|
||||||
|
return result
|
||||||
|
|
||||||
fetched_at: datetime = row[1]
|
fetched_at: datetime = row[1]
|
||||||
# Убедимся что timezone-aware для корректного сравнения
|
# Убедимся что timezone-aware для корректного сравнения
|
||||||
|
|
|
||||||
|
|
@ -125,11 +125,17 @@ _UPSERT_SQL = text(
|
||||||
harvest_error, region_code
|
harvest_error, region_code
|
||||||
) VALUES (
|
) VALUES (
|
||||||
:quarter_cad,
|
:quarter_cad,
|
||||||
CASE WHEN :geom_json IS NULL THEN NULL
|
CASE WHEN CAST(:geom_json AS text) IS NULL THEN NULL
|
||||||
ELSE ST_Multi(ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(:geom_json), 3857), 4326))
|
ELSE ST_Multi(ST_Transform(
|
||||||
|
ST_SetSRID(ST_GeomFromGeoJSON(CAST(:geom_json AS text)), 3857), 4326))
|
||||||
END,
|
END,
|
||||||
CASE WHEN :bbox_xmin IS NULL THEN NULL
|
CASE WHEN CAST(:bbox_xmin AS double precision) IS NULL THEN NULL
|
||||||
ELSE ST_MakeEnvelope(:bbox_xmin, :bbox_ymin, :bbox_xmax, :bbox_ymax, 3857)
|
ELSE ST_MakeEnvelope(
|
||||||
|
CAST(:bbox_xmin AS double precision),
|
||||||
|
CAST(:bbox_ymin AS double precision),
|
||||||
|
CAST(:bbox_xmax AS double precision),
|
||||||
|
CAST(:bbox_ymax AS double precision),
|
||||||
|
3857)
|
||||||
END,
|
END,
|
||||||
:parcels_count, :buildings_count, :territorial_zones_count,
|
:parcels_count, :buildings_count, :territorial_zones_count,
|
||||||
:red_lines_count, :engineering_count, :zouit_count, :risks_count, :total_features,
|
:red_lines_count, :engineering_count, :zouit_count, :risks_count, :total_features,
|
||||||
|
|
|
||||||
316
backend/tests/scrapers/test_nspd_grid_walk.py
Normal file
316
backend/tests/scrapers/test_nspd_grid_walk.py
Normal file
|
|
@ -0,0 +1,316 @@
|
||||||
|
"""Unit-тесты для NSPDClient.get_features_in_bbox_grid + classify_engineering_kind.
|
||||||
|
|
||||||
|
Sub-PR A foundation (#126): grid-walk — НЕ live HTTP (all mocked).
|
||||||
|
|
||||||
|
Запуск:
|
||||||
|
cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -v
|
||||||
|
|
||||||
|
Live integration (помечены @pytest.mark.integration — пропускаются в CI):
|
||||||
|
cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -m integration -s
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.services.scrapers.nspd_client import NSPDClient, NSPDFeature
|
||||||
|
from app.services.scrapers.nspd_denorm import classify_engineering_kind
|
||||||
|
|
||||||
|
# ── Helpers ────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _make_bulk_feature(feature_id: str | int | None, props: dict[str, Any]) -> MagicMock:
|
||||||
|
"""Создать mock NSPDBulkFeature с нужными полями."""
|
||||||
|
feat = MagicMock()
|
||||||
|
feat.id = feature_id
|
||||||
|
feat.geometry = None
|
||||||
|
feat.properties = props
|
||||||
|
return feat
|
||||||
|
|
||||||
|
|
||||||
|
# ── Grid-walk count tests ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetFeaturesInBboxGrid:
|
||||||
|
"""Тесты grid-walk метода (без live NSPD)."""
|
||||||
|
|
||||||
|
# bbox 700×700м в EPSG:3857 (типичный квартал ЕКБ)
|
||||||
|
BBOX: tuple[float, float, float, float] = (
|
||||||
|
6_600_000.0,
|
||||||
|
7_700_000.0,
|
||||||
|
6_600_700.0,
|
||||||
|
7_700_700.0,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _patch_bulk_client(self, return_features: list[list[Any]]) -> tuple[Any, AsyncMock]:
|
||||||
|
"""Патч NSPDBulkClient.wms_feature_info.
|
||||||
|
|
||||||
|
return_features: список ответов — по одному на каждый вызов wms_feature_info
|
||||||
|
(в порядке вызовов). Если список короче числа calls — последний элемент
|
||||||
|
повторяется.
|
||||||
|
"""
|
||||||
|
call_idx: list[int] = [0]
|
||||||
|
|
||||||
|
async def _side_effect(*args: Any, **kwargs: Any) -> list[Any]:
|
||||||
|
idx = min(call_idx[0], len(return_features) - 1)
|
||||||
|
call_idx[0] += 1
|
||||||
|
return return_features[idx]
|
||||||
|
|
||||||
|
mock_wms = AsyncMock(side_effect=_side_effect)
|
||||||
|
return mock_wms, mock_wms
|
||||||
|
|
||||||
|
def test_grid_n7_calls_49_times(self) -> None:
|
||||||
|
"""grid_n=7 → ровно 49 вызовов wms_feature_info."""
|
||||||
|
call_count: list[int] = [0]
|
||||||
|
|
||||||
|
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
|
||||||
|
call_count[0] += 1
|
||||||
|
return []
|
||||||
|
|
||||||
|
mock_client_instance = AsyncMock()
|
||||||
|
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
|
||||||
|
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
|
||||||
|
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
|
||||||
|
return_value=mock_client_instance,
|
||||||
|
):
|
||||||
|
client = NSPDClient()
|
||||||
|
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=7, step_m=1.0)
|
||||||
|
|
||||||
|
assert call_count[0] == 49, f"Ожидали 49 вызовов, получили {call_count[0]}"
|
||||||
|
assert result == []
|
||||||
|
|
||||||
|
def test_grid_n3_calls_9_times(self) -> None:
|
||||||
|
"""grid_n=3 → ровно 9 вызовов."""
|
||||||
|
call_count: list[int] = [0]
|
||||||
|
|
||||||
|
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
|
||||||
|
call_count[0] += 1
|
||||||
|
return []
|
||||||
|
|
||||||
|
mock_client_instance = AsyncMock()
|
||||||
|
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
|
||||||
|
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
|
||||||
|
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
|
||||||
|
return_value=mock_client_instance,
|
||||||
|
):
|
||||||
|
client = NSPDClient()
|
||||||
|
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=3, step_m=1.0)
|
||||||
|
|
||||||
|
assert call_count[0] == 9
|
||||||
|
assert result == []
|
||||||
|
|
||||||
|
def test_dedup_same_feature_id(self) -> None:
|
||||||
|
"""Одинаковые feature_id из соседних ячеек → одна запись."""
|
||||||
|
feat_a = _make_bulk_feature("feat-001", {"cad_num": "66:41:001:1"})
|
||||||
|
feat_b = _make_bulk_feature("feat-001", {"cad_num": "66:41:001:1"}) # дубликат
|
||||||
|
feat_c = _make_bulk_feature("feat-002", {"cad_num": "66:41:001:2"})
|
||||||
|
|
||||||
|
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
|
||||||
|
return [feat_a, feat_b, feat_c]
|
||||||
|
|
||||||
|
mock_client_instance = AsyncMock()
|
||||||
|
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
|
||||||
|
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
|
||||||
|
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
|
||||||
|
return_value=mock_client_instance,
|
||||||
|
):
|
||||||
|
client = NSPDClient()
|
||||||
|
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0)
|
||||||
|
|
||||||
|
# 4 cells × 3 features = 12 raw, но feature_id уникальных 2
|
||||||
|
feature_ids = [f.feature_id for f in result]
|
||||||
|
assert "feat-001" in feature_ids
|
||||||
|
assert "feat-002" in feature_ids
|
||||||
|
# нет дубликатов
|
||||||
|
assert len(feature_ids) == len(set(feature_ids))
|
||||||
|
|
||||||
|
def test_dedup_by_cad_num_when_no_feature_id(self) -> None:
|
||||||
|
"""Если feature_id=None — дедупликация по cad_num."""
|
||||||
|
feat_a = _make_bulk_feature(None, {"cad_num": "66:41:0000001:100"})
|
||||||
|
feat_b = _make_bulk_feature(None, {"cad_num": "66:41:0000001:100"}) # дубликат
|
||||||
|
|
||||||
|
call_n: list[int] = [0]
|
||||||
|
|
||||||
|
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
|
||||||
|
call_n[0] += 1
|
||||||
|
# Первая ячейка возвращает feat_a, вторая — feat_b
|
||||||
|
return [feat_a] if call_n[0] == 1 else [feat_b]
|
||||||
|
|
||||||
|
mock_client_instance = AsyncMock()
|
||||||
|
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
|
||||||
|
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
|
||||||
|
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
|
||||||
|
return_value=mock_client_instance,
|
||||||
|
):
|
||||||
|
client = NSPDClient()
|
||||||
|
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0)
|
||||||
|
|
||||||
|
cad_nums = [f.properties.get("cad_num") for f in result]
|
||||||
|
assert cad_nums.count("66:41:0000001:100") == 1, "Дубликат не дедуплицирован"
|
||||||
|
|
||||||
|
def test_error_in_one_cell_does_not_abort(self) -> None:
|
||||||
|
"""Ошибка в одной ячейке не останавливает весь grid-walk."""
|
||||||
|
good_feat = _make_bulk_feature("feat-ok", {"cad_num": "66:41:001:1"})
|
||||||
|
call_n: list[int] = [0]
|
||||||
|
|
||||||
|
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
|
||||||
|
call_n[0] += 1
|
||||||
|
if call_n[0] == 1:
|
||||||
|
raise RuntimeError("Simulated cell error")
|
||||||
|
return [good_feat]
|
||||||
|
|
||||||
|
mock_client_instance = AsyncMock()
|
||||||
|
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
|
||||||
|
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
|
||||||
|
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
|
||||||
|
return_value=mock_client_instance,
|
||||||
|
):
|
||||||
|
client = NSPDClient()
|
||||||
|
# Не должно бросать исключение
|
||||||
|
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=2, step_m=1.0)
|
||||||
|
|
||||||
|
# 4 cells: 1 error + 3 good_feat → 1 unique feature
|
||||||
|
assert any(f.feature_id == "feat-ok" for f in result)
|
||||||
|
|
||||||
|
def test_returns_nspd_feature_instances(self) -> None:
|
||||||
|
"""Метод возвращает list[NSPDFeature] а не NSPDBulkFeature."""
|
||||||
|
bulk_feat = _make_bulk_feature("feat-xyz", {"cad_num": "66:41:001:1"})
|
||||||
|
|
||||||
|
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
|
||||||
|
return [bulk_feat]
|
||||||
|
|
||||||
|
mock_client_instance = AsyncMock()
|
||||||
|
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
|
||||||
|
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
|
||||||
|
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
|
||||||
|
return_value=mock_client_instance,
|
||||||
|
):
|
||||||
|
client = NSPDClient()
|
||||||
|
result = client.get_features_in_bbox_grid(36328, self.BBOX, grid_n=1, step_m=1.0)
|
||||||
|
|
||||||
|
assert all(isinstance(f, NSPDFeature) for f in result)
|
||||||
|
|
||||||
|
def test_auto_reduce_grid_for_small_bbox(self) -> None:
|
||||||
|
"""Маленький bbox + большой grid_n + большой step_m → grid_n уменьшается."""
|
||||||
|
# bbox 60×60м, step_m=50, grid_n=7 → effective_n=min(7, int(60/50), int(60/50)) = 1
|
||||||
|
small_bbox: tuple[float, float, float, float] = (
|
||||||
|
6_600_000.0,
|
||||||
|
7_700_000.0,
|
||||||
|
6_600_060.0,
|
||||||
|
7_700_060.0,
|
||||||
|
)
|
||||||
|
call_count: list[int] = [0]
|
||||||
|
|
||||||
|
async def _wms(*args: Any, **kwargs: Any) -> list[Any]:
|
||||||
|
call_count[0] += 1
|
||||||
|
return []
|
||||||
|
|
||||||
|
mock_client_instance = AsyncMock()
|
||||||
|
mock_client_instance.wms_feature_info = AsyncMock(side_effect=_wms)
|
||||||
|
mock_client_instance.__aenter__ = AsyncMock(return_value=mock_client_instance)
|
||||||
|
mock_client_instance.__aexit__ = AsyncMock(return_value=None)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"app.scrapers.nspd_bulk_client.NSPDBulkClient",
|
||||||
|
return_value=mock_client_instance,
|
||||||
|
):
|
||||||
|
client = NSPDClient()
|
||||||
|
client.get_features_in_bbox_grid(36328, small_bbox, grid_n=7, step_m=50.0)
|
||||||
|
|
||||||
|
# effective_n = 1 → 1 ячейка
|
||||||
|
assert call_count[0] == 1
|
||||||
|
|
||||||
|
|
||||||
|
# ── Classifier tests ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestClassifyEngineeringKind:
|
||||||
|
"""Smoke-тесты классификатора инженерных сооружений."""
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"props, expected",
|
||||||
|
[
|
||||||
|
# gas
|
||||||
|
({"params_name": "Газопровод высокого давления"}, "gas"),
|
||||||
|
({"name": "Газопровод ПЭ SDR11 160мм"}, "gas"),
|
||||||
|
({"params_purpose": "Газоснабжение"}, "gas"),
|
||||||
|
({"name": "ГРП №12"}, "gas"),
|
||||||
|
# electric
|
||||||
|
({"name": "КЛ 10 кВ ТП 64102"}, "electric"),
|
||||||
|
({"params_name": "ВЛ-10кВ Ф-14"}, "electric"),
|
||||||
|
({"name": "ТП 1234"}, "electric"),
|
||||||
|
({"params_purpose": "Электроэнергетика и связь"}, "electric"),
|
||||||
|
({"name": "Подстанция 110/10 кВ"}, "electric"),
|
||||||
|
# water
|
||||||
|
({"params_purpose": "Водопровод хозбытовой"}, "water"),
|
||||||
|
({"name": "Водовод Ду300"}, "water"),
|
||||||
|
({"params_name": "Сеть водоснабжения"}, "water"),
|
||||||
|
# heat
|
||||||
|
({"params_name": "Тепловая сеть"}, "heat"),
|
||||||
|
({"name": "Теплосеть квартал 24"}, "heat"),
|
||||||
|
({"params_purpose": "Теплоснабжение жилых домов"}, "heat"),
|
||||||
|
({"name": "ТЭЦ-4 отпайка"}, "heat"),
|
||||||
|
# sewage
|
||||||
|
({"name": "Канализация"}, "sewage"),
|
||||||
|
({"params_name": "Сеть канализации Ду200"}, "sewage"),
|
||||||
|
({"name": "Ливневая канализация"}, "sewage"),
|
||||||
|
# other
|
||||||
|
({"name": "Объект не классифицирован"}, "other"),
|
||||||
|
({}, "other"),
|
||||||
|
({"params_purpose": None}, "other"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_classify(self, props: dict[str, Any], expected: str) -> None:
|
||||||
|
assert (
|
||||||
|
classify_engineering_kind(props) == expected
|
||||||
|
), f"props={props!r} → expected {expected!r}"
|
||||||
|
|
||||||
|
def test_field_priority_params_name_over_purpose(self) -> None:
|
||||||
|
"""params_name проверяется раньше purpose."""
|
||||||
|
props = {
|
||||||
|
"params_name": "Газопровод", # → gas
|
||||||
|
"params_purpose": "Водоснабжение", # → water (если бы проверялось первым)
|
||||||
|
}
|
||||||
|
# gas-паттерн найдётся в combined строке первым по порядку _ENGINEERING_PATTERNS
|
||||||
|
assert classify_engineering_kind(props) == "gas"
|
||||||
|
|
||||||
|
def test_case_insensitive(self) -> None:
|
||||||
|
"""Паттерны case-insensitive."""
|
||||||
|
assert classify_engineering_kind({"name": "ГАЗОПРОВОД ВЫСОКОГО ДАВЛЕНИЯ"}) == "gas"
|
||||||
|
assert classify_engineering_kind({"name": "канализация бытовая"}) == "sewage"
|
||||||
|
|
||||||
|
|
||||||
|
# ── Integration marker (skip in CI) ──────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
def test_live_nspd_grid_walk_skipped() -> None:
|
||||||
|
"""Placeholder для ручного запуска live NSPD grid-walk.
|
||||||
|
|
||||||
|
Запуск:
|
||||||
|
cd backend && uv run pytest tests/scrapers/test_nspd_grid_walk.py -m integration -s
|
||||||
|
|
||||||
|
При запуске — делает реальные HTTP-запросы к nspd.gov.ru.
|
||||||
|
"""
|
||||||
|
pytest.skip("Live NSPD integration test — запускать вручную с -m integration")
|
||||||
|
|
@ -11,6 +11,7 @@ from io import BytesIO
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
import httpx
|
||||||
import pytest
|
import pytest
|
||||||
from openpyxl import Workbook
|
from openpyxl import Workbook
|
||||||
|
|
||||||
|
|
@ -327,13 +328,28 @@ class TestDownloadXlsx:
|
||||||
client.download_xlsx(9999)
|
client.download_xlsx(9999)
|
||||||
|
|
||||||
def test_download_raises_on_http_error(self) -> None:
|
def test_download_raises_on_http_error(self) -> None:
|
||||||
import httpx as httpx_mod
|
|
||||||
|
|
||||||
mock_response = MagicMock()
|
mock_response = MagicMock()
|
||||||
mock_response.raise_for_status.side_effect = httpx_mod.HTTPStatusError(
|
mock_response.raise_for_status.side_effect = httpx.HTTPStatusError(
|
||||||
"404", request=MagicMock(), response=MagicMock()
|
"404", request=MagicMock(), response=MagicMock()
|
||||||
)
|
)
|
||||||
|
|
||||||
client = self._client_with_mock_http(mock_response)
|
client = self._client_with_mock_http(mock_response)
|
||||||
with pytest.raises(httpx_mod.HTTPStatusError):
|
with pytest.raises(httpx.HTTPStatusError):
|
||||||
client.download_xlsx(2026)
|
client.download_xlsx(2026)
|
||||||
|
|
||||||
|
|
||||||
|
# ── SSL verification test (Issue #242) ───────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestSslConfiguration:
|
||||||
|
def test_client_uses_verify_false(self) -> None:
|
||||||
|
"""EkburgPermitsClient должен создавать httpx.Client с verify=False.
|
||||||
|
|
||||||
|
екатеринбург.рф использует CA Минцифры РФ — не в certifi bundle.
|
||||||
|
Issue #242: SSL: CERTIFICATE_VERIFY_FAILED для всех 5 годов.
|
||||||
|
"""
|
||||||
|
with EkburgPermitsClient() as client:
|
||||||
|
# httpx.Client хранит ssl_context; verify=False создаёт unverified ctx
|
||||||
|
assert client._client._transport is not None
|
||||||
|
# Проверяем через атрибут _client напрямую — он должен быть httpx.Client
|
||||||
|
assert isinstance(client._client, httpx.Client)
|
||||||
|
|
|
||||||
|
|
@ -20,10 +20,12 @@ import pytest
|
||||||
from app.services.site_finder.quarter_dump_lookup import (
|
from app.services.site_finder.quarter_dump_lookup import (
|
||||||
EMPTY_DUMP_RESULT,
|
EMPTY_DUMP_RESULT,
|
||||||
_extract_features_by_layer,
|
_extract_features_by_layer,
|
||||||
|
_get_cad_zouit_overlaps,
|
||||||
_get_opportunity_parcels,
|
_get_opportunity_parcels,
|
||||||
_get_red_lines,
|
_get_red_lines,
|
||||||
_get_risk_zones,
|
_get_risk_zones,
|
||||||
derive_quarter_cad,
|
derive_quarter_cad,
|
||||||
|
get_quarter_dump_data,
|
||||||
make_empty_result,
|
make_empty_result,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -466,3 +468,153 @@ def test_acquire_harvest_lock_redis_unavailable_returns_false(
|
||||||
)
|
)
|
||||||
def test_derive_quarter_cad(cad: str, expected: str | None) -> None:
|
def test_derive_quarter_cad(cad: str, expected: str | None) -> None:
|
||||||
assert derive_quarter_cad(cad) == expected
|
assert derive_quarter_cad(cad) == expected
|
||||||
|
|
||||||
|
|
||||||
|
# ── #243: cad_zouit fallback when dump absent ─────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _make_mock_db_no_dump_with_cad_zouit(
|
||||||
|
cad_zouit_rows: list[tuple[Any, ...]],
|
||||||
|
) -> MagicMock:
|
||||||
|
"""Мок Session: nspd_quarter_dumps.first() → None; cad_zouit.fetchall() → rows."""
|
||||||
|
db = MagicMock()
|
||||||
|
first_result = MagicMock()
|
||||||
|
first_result.first.return_value = None # нет дампа
|
||||||
|
|
||||||
|
fetchall_result = MagicMock()
|
||||||
|
fetchall_result.fetchall.return_value = cad_zouit_rows
|
||||||
|
|
||||||
|
# Первый вызов execute (SELECT nspd_quarter_dumps) → first()=None
|
||||||
|
# Второй вызов execute (SELECT cad_zouit) → fetchall()=rows
|
||||||
|
db.execute.side_effect = [first_result, fetchall_result]
|
||||||
|
return db
|
||||||
|
|
||||||
|
|
||||||
|
def test_cad_zouit_fallback_fires_when_no_dump() -> None:
|
||||||
|
"""#243: cad_zouit fallback срабатывает когда nspd_quarter_dumps пуст глобально.
|
||||||
|
|
||||||
|
Ожидаем: nspd_zouit_overlaps непустой, source='cad_zouit'.
|
||||||
|
"""
|
||||||
|
cad_zouit_rows: list[tuple[Any, ...]] = [
|
||||||
|
# (type_zone, category_name, name_by_doc, reg_numb_border, id)
|
||||||
|
("Охранная зона трубопровода", "Трубопровод", "ГВС ул. Ленина", "66-66-00/123", 42),
|
||||||
|
]
|
||||||
|
db = _make_mock_db_no_dump_with_cad_zouit(cad_zouit_rows)
|
||||||
|
|
||||||
|
wkt = "POLYGON((60 56,61 56,61 57,60 57,60 56))"
|
||||||
|
result = get_quarter_dump_data(db, "66:41:0603016:194", wkt)
|
||||||
|
|
||||||
|
overlaps: list[dict[str, Any]] = result["nspd_zouit_overlaps"]
|
||||||
|
assert len(overlaps) == 1
|
||||||
|
assert overlaps[0]["source"] == "cad_zouit"
|
||||||
|
assert overlaps[0]["type_zone"] == "Охранная зона трубопровода"
|
||||||
|
assert overlaps[0]["name"] == "ГВС ул. Ленина"
|
||||||
|
# dump.available остаётся False (нет дампа)
|
||||||
|
assert result["nspd_dump"]["available"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_cad_zouit_fallback_no_dump_no_parcel_wkt() -> None:
|
||||||
|
"""#243: dump=None + parcel_wkt=None → overlaps пустой, не вызываем cad_zouit."""
|
||||||
|
db = MagicMock()
|
||||||
|
first_result = MagicMock()
|
||||||
|
first_result.first.return_value = None
|
||||||
|
db.execute.return_value = first_result
|
||||||
|
|
||||||
|
result = get_quarter_dump_data(db, "66:41:0603016:194", None)
|
||||||
|
|
||||||
|
assert result["nspd_zouit_overlaps"] == []
|
||||||
|
# cad_zouit SELECT не должен вызываться (только один execute для dump lookup)
|
||||||
|
assert db.execute.call_count == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_cad_zouit_fallback_empty_when_no_overlaps() -> None:
|
||||||
|
"""#243: dump=None + cad_zouit возвращает 0 строк → nspd_zouit_overlaps пуст."""
|
||||||
|
db = _make_mock_db_no_dump_with_cad_zouit([])
|
||||||
|
|
||||||
|
wkt = "POLYGON((60 56,61 56,61 57,60 57,60 56))"
|
||||||
|
result = get_quarter_dump_data(db, "66:41:0603016:194", wkt)
|
||||||
|
|
||||||
|
assert result["nspd_zouit_overlaps"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_cad_zouit_overlaps_returns_correct_schema() -> None:
|
||||||
|
"""_get_cad_zouit_overlaps: поля group_key/layer/source/type_zone корректны."""
|
||||||
|
rows: list[tuple[Any, ...]] = [
|
||||||
|
# (type_zone, category_name, name_by_doc, reg_numb_border, id)
|
||||||
|
("Охранная зона ЛЭП 110кВ", "Электроснабжение", "ВЛ-110", "66-123", 7),
|
||||||
|
("СЗЗ промышленного объекта", "Санитарная", "Завод", "66-456", 8),
|
||||||
|
]
|
||||||
|
db = _make_mock_db_with_rows(rows)
|
||||||
|
|
||||||
|
result = _get_cad_zouit_overlaps(db, "POLYGON((0 0,1 0,1 1,0 0))")
|
||||||
|
|
||||||
|
assert len(result) == 2
|
||||||
|
lep = result[0]
|
||||||
|
assert lep["group_key"] == "cad_zouit"
|
||||||
|
assert lep["source"] == "cad_zouit"
|
||||||
|
assert lep["type_zone"] == "Охранная зона ЛЭП 110кВ"
|
||||||
|
assert lep["subcategory"] is None # всегда NULL в cad_zouit
|
||||||
|
assert lep["name"] == "ВЛ-110"
|
||||||
|
assert lep["raw_props"]["zouit_id"] == 7
|
||||||
|
|
||||||
|
|
||||||
|
def test_nspd_path_used_when_zouit_count_nonzero() -> None:
|
||||||
|
"""#243 backward compat: когда dump существует с zouit_count>0 — NSPD path, не cad_zouit.
|
||||||
|
|
||||||
|
Мок: row.zouit_count=3, fetchall возвращает NSPD features.
|
||||||
|
Проверяем source='nspd-quarter-dump' (не cad_zouit).
|
||||||
|
"""
|
||||||
|
# Строка дампа: все поля через MagicMock
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
dump_row = MagicMock()
|
||||||
|
dump_row.__getitem__ = lambda self, i: [
|
||||||
|
"66:41:0603016", # quarter_cad
|
||||||
|
datetime(2026, 1, 1, tzinfo=UTC), # fetched_at_utc (свежий)
|
||||||
|
100, # total_features
|
||||||
|
None, # harvest_error
|
||||||
|
1, # territorial_zones_count
|
||||||
|
3, # zouit_count > 0 → NSPD path
|
||||||
|
0, # engineering_count
|
||||||
|
0, # risks_count
|
||||||
|
0, # opportunity_count
|
||||||
|
0, # red_lines_count
|
||||||
|
][i]
|
||||||
|
|
||||||
|
nspd_zouit_rows: list[tuple[Any, ...]] = [
|
||||||
|
# (layer, props)
|
||||||
|
("zouit_engineering", {"name": "ЛЭП", "subcategory": 17}),
|
||||||
|
]
|
||||||
|
|
||||||
|
db = MagicMock()
|
||||||
|
first_result = MagicMock()
|
||||||
|
first_result.first.return_value = dump_row
|
||||||
|
|
||||||
|
# Последующие вызовы execute (spatial queries) — возвращаем пустые кроме zouit
|
||||||
|
zouit_result = MagicMock()
|
||||||
|
zouit_result.fetchall.return_value = nspd_zouit_rows
|
||||||
|
|
||||||
|
empty_result = MagicMock()
|
||||||
|
empty_result.fetchall.return_value = []
|
||||||
|
empty_result.first.return_value = None
|
||||||
|
|
||||||
|
# [0]=dumps lookup, [1]=zoning(first), [2]=zouit(fetchall), [3+]=others
|
||||||
|
db.execute.side_effect = [
|
||||||
|
first_result,
|
||||||
|
empty_result, # _get_zoning first()
|
||||||
|
zouit_result, # _get_zouit_overlaps fetchall()
|
||||||
|
empty_result, # _get_engineering_nearby fetchall()
|
||||||
|
empty_result, # _get_risk_zones fetchall()
|
||||||
|
empty_result, # _get_opportunity_parcels fetchall()
|
||||||
|
empty_result, # _get_red_lines fetchall()
|
||||||
|
]
|
||||||
|
|
||||||
|
result = get_quarter_dump_data(
|
||||||
|
db, "66:41:0603016:194", "POLYGON((60 56,61 56,61 57,60 57,60 56))"
|
||||||
|
)
|
||||||
|
|
||||||
|
overlaps: list[dict[str, Any]] = result["nspd_zouit_overlaps"]
|
||||||
|
assert len(overlaps) == 1
|
||||||
|
assert overlaps[0]["source"] == "nspd-quarter-dump"
|
||||||
|
# dump.available=True т.к. свежий дамп
|
||||||
|
assert result["nspd_dump"]["available"] is True
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,7 @@ from app.workers.tasks.nspd_sync import (
|
||||||
_build_features_json,
|
_build_features_json,
|
||||||
_build_risks_count,
|
_build_risks_count,
|
||||||
_build_zouit_count,
|
_build_zouit_count,
|
||||||
|
_upsert_dump,
|
||||||
harvest_quarter,
|
harvest_quarter,
|
||||||
harvest_stale_quarters,
|
harvest_stale_quarters,
|
||||||
)
|
)
|
||||||
|
|
@ -71,6 +72,7 @@ def _make_dump(
|
||||||
engineering_structures=[_make_feat("e1")],
|
engineering_structures=[_make_feat("e1")],
|
||||||
zouit=zouit,
|
zouit=zouit,
|
||||||
risks=risks,
|
risks=risks,
|
||||||
|
opportunity={},
|
||||||
layers_fetched=(
|
layers_fetched=(
|
||||||
"search",
|
"search",
|
||||||
"parcels",
|
"parcels",
|
||||||
|
|
@ -204,6 +206,7 @@ def test_harvest_quarter_empty_quarter(
|
||||||
engineering_structures=[],
|
engineering_structures=[],
|
||||||
zouit={"okn": [], "engineering": [], "natural": [], "protected": [], "other": []},
|
zouit={"okn": [], "engineering": [], "natural": [], "protected": [], "other": []},
|
||||||
risks={},
|
risks={},
|
||||||
|
opportunity={},
|
||||||
layers_fetched=("search",),
|
layers_fetched=("search",),
|
||||||
bbox_3857=None,
|
bbox_3857=None,
|
||||||
fetched_at_utc="2026-05-12T03:00:00+00:00",
|
fetched_at_utc="2026-05-12T03:00:00+00:00",
|
||||||
|
|
@ -365,6 +368,7 @@ def test_features_json_geometry_preserved() -> None:
|
||||||
engineering_structures=[],
|
engineering_structures=[],
|
||||||
zouit={},
|
zouit={},
|
||||||
risks={},
|
risks={},
|
||||||
|
opportunity={},
|
||||||
layers_fetched=("search", "parcels"),
|
layers_fetched=("search", "parcels"),
|
||||||
bbox_3857=(100.0, 200.0, 300.0, 400.0),
|
bbox_3857=(100.0, 200.0, 300.0, 400.0),
|
||||||
fetched_at_utc="2026-05-12T00:00:00+00:00",
|
fetched_at_utc="2026-05-12T00:00:00+00:00",
|
||||||
|
|
@ -375,3 +379,70 @@ def test_features_json_geometry_preserved() -> None:
|
||||||
assert result[0]["layer"] == "parcels"
|
assert result[0]["layer"] == "parcels"
|
||||||
assert result[0]["feature_id"] == "p1"
|
assert result[0]["feature_id"] == "p1"
|
||||||
assert result[0]["properties"] == {"k": "v"}
|
assert result[0]["properties"] == {"k": "v"}
|
||||||
|
|
||||||
|
|
||||||
|
# ── _upsert_dump CAST regression tests (#244) ────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@patch("app.workers.tasks.nspd_sync.SessionLocal")
|
||||||
|
def test_upsert_dump_null_geom_executes_without_error(mock_session_cls: MagicMock) -> None:
|
||||||
|
"""Regression #244: _upsert_dump с geom_json=None не падает на AmbiguousParameter.
|
||||||
|
|
||||||
|
psycopg3 не может вывести тип голого параметра $N внутри CASE WHEN ... IS NULL.
|
||||||
|
Fix: CAST(:geom_json AS text) IS NULL даёт явный тип — нет AmbiguousParameter.
|
||||||
|
"""
|
||||||
|
mock_db = MagicMock()
|
||||||
|
mock_session_cls.return_value = mock_db
|
||||||
|
|
||||||
|
# dump=None → error-only path: все geo-параметры None, должен execute без ошибок
|
||||||
|
_upsert_dump(
|
||||||
|
quarter_cad="66:41:0204016",
|
||||||
|
region_code=66,
|
||||||
|
dump=None,
|
||||||
|
features_json=None,
|
||||||
|
duration_ms=100,
|
||||||
|
harvest_error="TestError: simulated",
|
||||||
|
)
|
||||||
|
|
||||||
|
mock_db.execute.assert_called_once()
|
||||||
|
mock_db.commit.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
@patch("app.workers.tasks.nspd_sync.SessionLocal")
|
||||||
|
def test_upsert_dump_with_geom_executes_without_error(mock_session_cls: MagicMock) -> None:
|
||||||
|
"""Regression #244: _upsert_dump с реальным geom_json и bbox не падает.
|
||||||
|
|
||||||
|
CAST(:geom_json AS text) передаёт строку, CAST(:bbox_xmin AS double precision) —
|
||||||
|
число. Оба branch CASE WHEN типизированы корректно.
|
||||||
|
"""
|
||||||
|
mock_db = MagicMock()
|
||||||
|
mock_session_cls.return_value = mock_db
|
||||||
|
|
||||||
|
geom = {
|
||||||
|
"type": "Polygon",
|
||||||
|
"coordinates": [
|
||||||
|
[[100.0, 200.0], [300.0, 200.0], [300.0, 400.0], [100.0, 400.0], [100.0, 200.0]]
|
||||||
|
],
|
||||||
|
}
|
||||||
|
dump = _make_dump(
|
||||||
|
quarter_cad="66:41:0204016",
|
||||||
|
quarter_feat=_make_feat_with_geom("q1", geom),
|
||||||
|
bbox=(100.0, 200.0, 300.0, 400.0),
|
||||||
|
)
|
||||||
|
|
||||||
|
_upsert_dump(
|
||||||
|
quarter_cad="66:41:0204016",
|
||||||
|
region_code=66,
|
||||||
|
dump=dump,
|
||||||
|
features_json=_build_features_json(dump),
|
||||||
|
duration_ms=500,
|
||||||
|
harvest_error=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
mock_db.execute.assert_called_once()
|
||||||
|
# Verify params contain properly-typed values (not None for geo fields)
|
||||||
|
call_params = mock_db.execute.call_args[0][1]
|
||||||
|
assert call_params["geom_json"] is not None # JSON string
|
||||||
|
assert call_params["bbox_xmin"] == 100.0
|
||||||
|
assert call_params["bbox_ymax"] == 400.0
|
||||||
|
mock_db.commit.assert_called_once()
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue