448 lines
17 KiB
Python
448 lines
17 KiB
Python
"""Unit tests для nspd_sync Celery tasks (#94 Sprint 1.1 item #3).
|
||
|
||
Без сетевых запросов и реальной БД — mock-based. Проверяет:
|
||
- harvest_quarter happy path (correct upsert params, return dict)
|
||
- harvest_quarter empty quarter (NSPD не вернул квартал — empty row, no crash)
|
||
- harvest_quarter WAF retry (NspdLiteWafError → task re-raises → Celery autoretry)
|
||
- harvest_quarter generic exception (harvest_error row written)
|
||
- harvest_stale_quarters fanout (apply_async N times)
|
||
- _build_features_json layers completeness (all 5 ЗОУИТ присутствуют)
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from typing import Any
|
||
from unittest.mock import MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.nspd_client import (
|
||
NSPDFeature,
|
||
NspdLiteWafError,
|
||
QuarterDump,
|
||
)
|
||
from app.workers.tasks.nspd_sync import (
|
||
_build_features_json,
|
||
_build_risks_count,
|
||
_build_zouit_count,
|
||
_upsert_dump,
|
||
harvest_quarter,
|
||
harvest_stale_quarters,
|
||
)
|
||
|
||
# ── Fixtures ──────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _make_feat(fid: str = "x") -> NSPDFeature:
|
||
return NSPDFeature.from_raw({"id": fid, "geometry": None, "properties": {}})
|
||
|
||
|
||
def _make_feat_with_geom(fid: str, geom: dict[str, Any]) -> NSPDFeature:
|
||
return NSPDFeature.from_raw({"id": fid, "geometry": geom, "properties": {"k": "v"}})
|
||
|
||
|
||
def _make_dump(
|
||
quarter_cad: str = "66:41:0204016",
|
||
include_zouit: bool = True,
|
||
include_risks: bool = False,
|
||
quarter_feat: NSPDFeature | None = None,
|
||
bbox: tuple[float, float, float, float] | None = (100.0, 200.0, 300.0, 400.0),
|
||
) -> QuarterDump:
|
||
"""Фабрика QuarterDump для тестов."""
|
||
zouit: dict[str, list[NSPDFeature]] = {}
|
||
if include_zouit:
|
||
zouit = {
|
||
"okn": [_make_feat("ok1"), _make_feat("ok2")],
|
||
"engineering": [_make_feat("eng1")],
|
||
"natural": [],
|
||
"protected": [_make_feat("prot1")],
|
||
"other": [],
|
||
}
|
||
risks: dict[str, list[NSPDFeature]] = {}
|
||
if include_risks:
|
||
risks = {"flooding": [_make_feat("fl1")], "landslide": []}
|
||
|
||
return QuarterDump(
|
||
quarter_cad=quarter_cad,
|
||
quarter=quarter_feat,
|
||
parcels=[_make_feat("p1"), _make_feat("p2")],
|
||
buildings=[_make_feat("b1")],
|
||
territorial_zones=[_make_feat("tz1")],
|
||
red_lines=[],
|
||
engineering_structures=[_make_feat("e1")],
|
||
zouit=zouit,
|
||
risks=risks,
|
||
opportunity={},
|
||
layers_fetched=(
|
||
"search",
|
||
"parcels",
|
||
"buildings",
|
||
"territorial_zones",
|
||
"red_lines",
|
||
"engineering_structures",
|
||
),
|
||
bbox_3857=bbox,
|
||
fetched_at_utc="2026-05-12T03:00:00+00:00",
|
||
)
|
||
|
||
|
||
# ── _build_features_json tests ────────────────────────────────────────────────
|
||
|
||
|
||
def test_features_json_layers_all_present_with_zouit() -> None:
|
||
"""Если include_zouit=True — все 5 ЗОУИТ-групп присутствуют в features_json."""
|
||
dump = _make_dump(include_zouit=True)
|
||
result = _build_features_json(dump)
|
||
layers = {item["layer"] for item in result}
|
||
# Core layers
|
||
assert "parcels" in layers
|
||
assert "buildings" in layers
|
||
assert "territorial_zones" in layers
|
||
assert "engineering_structures" in layers
|
||
# ЗОУИТ — все 5 ключей (даже с пустыми списками они не попадут — только
|
||
# populated items)
|
||
# okn: 2, engineering: 1, natural: 0, protected: 1, other: 0
|
||
assert "zouit_okn" in layers
|
||
assert "zouit_engineering" in layers
|
||
assert "zouit_protected" in layers
|
||
# natural и other — пустые, не должны быть в layers
|
||
assert "zouit_natural" not in layers
|
||
assert "zouit_other" not in layers
|
||
|
||
|
||
def test_features_json_no_zouit_no_risk_layers() -> None:
|
||
"""Без include_zouit — zouit_* и risk_* ключей нет в features_json."""
|
||
dump = _make_dump(include_zouit=False, include_risks=False)
|
||
result = _build_features_json(dump)
|
||
layers = {item["layer"] for item in result}
|
||
for layer_name in layers:
|
||
assert not layer_name.startswith("zouit_")
|
||
assert not layer_name.startswith("risk_")
|
||
|
||
|
||
def test_features_json_each_feature_has_required_keys() -> None:
|
||
"""Каждый элемент features_json содержит layer, feature_id, geometry, properties."""
|
||
dump = _make_dump(include_zouit=True)
|
||
result = _build_features_json(dump)
|
||
for item in result:
|
||
assert "layer" in item
|
||
assert "feature_id" in item
|
||
assert "geometry" in item
|
||
assert "properties" in item
|
||
|
||
|
||
def test_build_zouit_count_sums_correctly() -> None:
|
||
"""_build_zouit_count суммирует все ЗОУИТ-группы."""
|
||
dump = _make_dump(include_zouit=True)
|
||
# okn:2 + engineering:1 + natural:0 + protected:1 + other:0 = 4
|
||
assert _build_zouit_count(dump) == 4
|
||
|
||
|
||
def test_build_risks_count_empty() -> None:
|
||
"""_build_risks_count при пустом risks dict = 0."""
|
||
dump = _make_dump(include_risks=False)
|
||
assert _build_risks_count(dump) == 0
|
||
|
||
|
||
def test_build_risks_count_with_risks() -> None:
|
||
"""_build_risks_count суммирует risk-группы."""
|
||
dump = _make_dump(include_zouit=False, include_risks=True)
|
||
# flooding:1 + landslide:0 = 1
|
||
assert _build_risks_count(dump) == 1
|
||
|
||
|
||
# ── harvest_quarter tests ─────────────────────────────────────────────────────
|
||
|
||
|
||
@patch("app.workers.tasks.nspd_sync._upsert_dump")
|
||
@patch("app.workers.tasks.nspd_sync.NSPDClient")
|
||
def test_harvest_quarter_success(
|
||
mock_client_cls: MagicMock,
|
||
mock_upsert: MagicMock,
|
||
) -> None:
|
||
"""Happy path: search_by_quarter → return populated QuarterDump → upsert called."""
|
||
dump = _make_dump(include_zouit=True)
|
||
mock_client = MagicMock()
|
||
mock_client.search_by_quarter.return_value = dump
|
||
mock_client_cls.return_value = mock_client
|
||
|
||
result = harvest_quarter(
|
||
"66:41:0204016",
|
||
region_code=66,
|
||
include_zouit=True,
|
||
include_risks=False,
|
||
)
|
||
|
||
assert result["quarter_cad"] == "66:41:0204016"
|
||
assert result["region_code"] == 66
|
||
assert result["total_features"] == dump.total_features
|
||
assert "harvest_duration_ms" in result
|
||
assert "error" not in result
|
||
|
||
# Verify _upsert_dump called once with correct positional args
|
||
# Signature: (quarter_cad, region_code, dump, features_json, duration_ms, harvest_error)
|
||
assert mock_upsert.call_count == 1
|
||
upsert_call = mock_upsert.call_args
|
||
assert upsert_call[0][0] == "66:41:0204016" # quarter_cad
|
||
assert upsert_call[0][1] == 66 # region_code
|
||
assert upsert_call[0][2] is dump # dump object
|
||
assert upsert_call[0][5] is None # harvest_error=None on success
|
||
|
||
|
||
@patch("app.workers.tasks.nspd_sync._upsert_dump")
|
||
@patch("app.workers.tasks.nspd_sync.NSPDClient")
|
||
def test_harvest_quarter_empty_quarter(
|
||
mock_client_cls: MagicMock,
|
||
mock_upsert: MagicMock,
|
||
) -> None:
|
||
"""Quarter не найден в NSPD (quarter=None, bbox=None) — empty harvest, no crash."""
|
||
empty_dump = QuarterDump(
|
||
quarter_cad="66:41:9999999",
|
||
quarter=None,
|
||
parcels=[],
|
||
buildings=[],
|
||
territorial_zones=[],
|
||
red_lines=[],
|
||
engineering_structures=[],
|
||
zouit={"okn": [], "engineering": [], "natural": [], "protected": [], "other": []},
|
||
risks={},
|
||
opportunity={},
|
||
layers_fetched=("search",),
|
||
bbox_3857=None,
|
||
fetched_at_utc="2026-05-12T03:00:00+00:00",
|
||
)
|
||
mock_client = MagicMock()
|
||
mock_client.search_by_quarter.return_value = empty_dump
|
||
mock_client_cls.return_value = mock_client
|
||
|
||
result = harvest_quarter("66:41:9999999", region_code=66)
|
||
|
||
# Should succeed with zero counts
|
||
assert result["quarter_cad"] == "66:41:9999999"
|
||
assert result["total_features"] == 0
|
||
assert "error" not in result
|
||
|
||
# Upsert called with the empty dump (not None)
|
||
# Signature: (quarter_cad, region_code, dump, features_json, duration_ms, harvest_error)
|
||
assert mock_upsert.call_count == 1
|
||
upsert_args = mock_upsert.call_args[0]
|
||
assert upsert_args[2] is empty_dump # dump passed
|
||
assert upsert_args[5] is None # harvest_error=None (valid empty harvest)
|
||
|
||
# features_json should be empty list
|
||
features_json_arg = upsert_args[3]
|
||
assert features_json_arg == []
|
||
|
||
|
||
@patch("app.workers.tasks.nspd_sync._upsert_dump")
|
||
@patch("app.workers.tasks.nspd_sync.NSPDClient")
|
||
def test_harvest_quarter_waf_retry(
|
||
mock_client_cls: MagicMock,
|
||
mock_upsert: MagicMock,
|
||
) -> None:
|
||
"""NspdLiteWafError → task re-raises (Celery autoretry_for будет retry).
|
||
|
||
autoretry_for=(NspdLiteWafError,) настроен в декораторе задачи.
|
||
Тест проверяет что re-raise происходит (task не глотает WAF).
|
||
"""
|
||
mock_client = MagicMock()
|
||
mock_client.search_by_quarter.side_effect = NspdLiteWafError("HTTP 429 WAF")
|
||
mock_client_cls.return_value = mock_client
|
||
|
||
with pytest.raises(NspdLiteWafError):
|
||
harvest_quarter("66:41:0204016", region_code=66)
|
||
|
||
# _upsert_dump НЕ должен быть вызван при WAF (не пишем error row, retry сделает попытку)
|
||
mock_upsert.assert_not_called()
|
||
|
||
|
||
@patch("app.workers.tasks.nspd_sync._upsert_dump")
|
||
@patch("app.workers.tasks.nspd_sync.NSPDClient")
|
||
def test_harvest_quarter_writes_error_row(
|
||
mock_client_cls: MagicMock,
|
||
mock_upsert: MagicMock,
|
||
) -> None:
|
||
"""Generic exception → error row inserted с harvest_error set; task НЕ raise."""
|
||
mock_client = MagicMock()
|
||
mock_client.search_by_quarter.side_effect = ConnectionError("timeout")
|
||
mock_client_cls.return_value = mock_client
|
||
|
||
result = harvest_quarter("66:41:0204016", region_code=66)
|
||
|
||
# Task returns error dict instead of raising
|
||
assert "error" in result
|
||
assert "ConnectionError" in result["error"]
|
||
assert result["quarter_cad"] == "66:41:0204016"
|
||
|
||
# _upsert_dump called with dump=None (error-only row)
|
||
# Signature: (quarter_cad, region_code, dump, features_json, duration_ms, harvest_error)
|
||
assert mock_upsert.call_count == 1
|
||
upsert_args = mock_upsert.call_args[0]
|
||
assert upsert_args[2] is None # dump=None
|
||
assert upsert_args[3] is None # features_json=None
|
||
assert upsert_args[5] is not None # harvest_error set
|
||
assert "ConnectionError" in upsert_args[5]
|
||
|
||
|
||
# ── harvest_stale_quarters tests ───────────────────────────────────────────────
|
||
|
||
|
||
@patch("app.workers.tasks.nspd_sync.harvest_quarter")
|
||
@patch("app.workers.tasks.nspd_sync.SessionLocal")
|
||
def test_harvest_stale_quarters_fanout(
|
||
mock_session_cls: MagicMock,
|
||
mock_task: MagicMock,
|
||
) -> None:
|
||
"""Stale cads найдены → harvest_quarter.apply_async вызван N раз."""
|
||
stale_cads = ["66:41:0000001", "66:41:0000002", "66:41:0000003"]
|
||
|
||
# Mock DB session
|
||
mock_db = MagicMock()
|
||
mock_session_cls.return_value = mock_db
|
||
mock_db.execute.return_value.all.return_value = [(c,) for c in stale_cads]
|
||
|
||
# harvest_quarter.apply_async — проверяем что вызван для каждого cad
|
||
mock_task.apply_async = MagicMock()
|
||
|
||
result = harvest_stale_quarters(region_code=66, max_age_days=90, batch_size=50)
|
||
|
||
assert result["enqueued"] == 3
|
||
assert result["stale_found"] == 3
|
||
assert result["region_code"] == 66
|
||
|
||
assert mock_task.apply_async.call_count == 3
|
||
# apply_async(args=[cad, region_code]) — args passed as keyword 'args'
|
||
called_cads = [c.kwargs["args"][0] for c in mock_task.apply_async.call_args_list]
|
||
assert called_cads == stale_cads
|
||
|
||
|
||
@patch("app.workers.tasks.nspd_sync.harvest_quarter")
|
||
@patch("app.workers.tasks.nspd_sync.SessionLocal")
|
||
def test_harvest_stale_quarters_empty(
|
||
mock_session_cls: MagicMock,
|
||
mock_task: MagicMock,
|
||
) -> None:
|
||
"""Нет stale кварталов → 0 enqueued, apply_async не вызван."""
|
||
mock_db = MagicMock()
|
||
mock_session_cls.return_value = mock_db
|
||
mock_db.execute.return_value.all.return_value = []
|
||
|
||
mock_task.apply_async = MagicMock()
|
||
|
||
result = harvest_stale_quarters(region_code=66)
|
||
|
||
assert result["enqueued"] == 0
|
||
assert result["stale_found"] == 0
|
||
mock_task.apply_async.assert_not_called()
|
||
|
||
|
||
# ── features_json layer names tests ──────────────────────────────────────────
|
||
|
||
|
||
def test_features_json_risk_layer_names() -> None:
|
||
"""Risk layers кодируются как risk_<short_name>."""
|
||
dump = _make_dump(include_zouit=False, include_risks=True)
|
||
result = _build_features_json(dump)
|
||
layers = {item["layer"] for item in result}
|
||
# flooding:1 item
|
||
assert "risk_flooding" in layers
|
||
# landslide:0 items — не должно быть ключа
|
||
assert "risk_landslide" not in layers
|
||
|
||
|
||
def test_features_json_geometry_preserved() -> None:
|
||
"""Geometry (EPSG:3857) сохраняется без трансформации в features_json."""
|
||
geom = {
|
||
"type": "Polygon",
|
||
"coordinates": [
|
||
[[100.0, 200.0], [300.0, 200.0], [300.0, 400.0], [100.0, 400.0], [100.0, 200.0]]
|
||
],
|
||
}
|
||
dump = QuarterDump(
|
||
quarter_cad="66:41:0204016",
|
||
quarter=None,
|
||
parcels=[_make_feat_with_geom("p1", geom)],
|
||
buildings=[],
|
||
territorial_zones=[],
|
||
red_lines=[],
|
||
engineering_structures=[],
|
||
zouit={},
|
||
risks={},
|
||
opportunity={},
|
||
layers_fetched=("search", "parcels"),
|
||
bbox_3857=(100.0, 200.0, 300.0, 400.0),
|
||
fetched_at_utc="2026-05-12T00:00:00+00:00",
|
||
)
|
||
result = _build_features_json(dump)
|
||
assert len(result) == 1
|
||
assert result[0]["geometry"] == geom # not transformed
|
||
assert result[0]["layer"] == "parcels"
|
||
assert result[0]["feature_id"] == "p1"
|
||
assert result[0]["properties"] == {"k": "v"}
|
||
|
||
|
||
# ── _upsert_dump CAST regression tests (#244) ────────────────────────────────
|
||
|
||
|
||
@patch("app.workers.tasks.nspd_sync.SessionLocal")
|
||
def test_upsert_dump_null_geom_executes_without_error(mock_session_cls: MagicMock) -> None:
|
||
"""Regression #244: _upsert_dump с geom_json=None не падает на AmbiguousParameter.
|
||
|
||
psycopg3 не может вывести тип голого параметра $N внутри CASE WHEN ... IS NULL.
|
||
Fix: CAST(:geom_json AS text) IS NULL даёт явный тип — нет AmbiguousParameter.
|
||
"""
|
||
mock_db = MagicMock()
|
||
mock_session_cls.return_value = mock_db
|
||
|
||
# dump=None → error-only path: все geo-параметры None, должен execute без ошибок
|
||
_upsert_dump(
|
||
quarter_cad="66:41:0204016",
|
||
region_code=66,
|
||
dump=None,
|
||
features_json=None,
|
||
duration_ms=100,
|
||
harvest_error="TestError: simulated",
|
||
)
|
||
|
||
mock_db.execute.assert_called_once()
|
||
mock_db.commit.assert_called_once()
|
||
|
||
|
||
@patch("app.workers.tasks.nspd_sync.SessionLocal")
|
||
def test_upsert_dump_with_geom_executes_without_error(mock_session_cls: MagicMock) -> None:
|
||
"""Regression #244: _upsert_dump с реальным geom_json и bbox не падает.
|
||
|
||
CAST(:geom_json AS text) передаёт строку, CAST(:bbox_xmin AS double precision) —
|
||
число. Оба branch CASE WHEN типизированы корректно.
|
||
"""
|
||
mock_db = MagicMock()
|
||
mock_session_cls.return_value = mock_db
|
||
|
||
geom = {
|
||
"type": "Polygon",
|
||
"coordinates": [
|
||
[[100.0, 200.0], [300.0, 200.0], [300.0, 400.0], [100.0, 400.0], [100.0, 200.0]]
|
||
],
|
||
}
|
||
dump = _make_dump(
|
||
quarter_cad="66:41:0204016",
|
||
quarter_feat=_make_feat_with_geom("q1", geom),
|
||
bbox=(100.0, 200.0, 300.0, 400.0),
|
||
)
|
||
|
||
_upsert_dump(
|
||
quarter_cad="66:41:0204016",
|
||
region_code=66,
|
||
dump=dump,
|
||
features_json=_build_features_json(dump),
|
||
duration_ms=500,
|
||
harvest_error=None,
|
||
)
|
||
|
||
mock_db.execute.assert_called_once()
|
||
# Verify params contain properly-typed values (not None for geo fields)
|
||
call_params = mock_db.execute.call_args[0][1]
|
||
assert call_params["geom_json"] is not None # JSON string
|
||
assert call_params["bbox_xmin"] == 100.0
|
||
assert call_params["bbox_ymax"] == 400.0
|
||
mock_db.commit.assert_called_once()
|