"""Unit tests для nspd_sync Celery tasks (#94 Sprint 1.1 item #3). Без сетевых запросов и реальной БД — mock-based. Проверяет: - harvest_quarter happy path (correct upsert params, return dict) - harvest_quarter empty quarter (NSPD не вернул квартал — empty row, no crash) - harvest_quarter WAF retry (NspdLiteWafError → task re-raises → Celery autoretry) - harvest_quarter generic exception (harvest_error row written) - harvest_stale_quarters fanout (apply_async N times) - _build_features_json layers completeness (all 5 ЗОУИТ присутствуют) """ from __future__ import annotations from typing import Any from unittest.mock import MagicMock, patch import pytest from app.services.scrapers.nspd_client import ( NSPDFeature, NspdLiteWafError, QuarterDump, ) from app.workers.tasks.nspd_sync import ( _build_features_json, _build_risks_count, _build_zouit_count, _upsert_dump, harvest_quarter, harvest_stale_quarters, ) # ── Fixtures ────────────────────────────────────────────────────────────────── def _make_feat(fid: str = "x") -> NSPDFeature: return NSPDFeature.from_raw({"id": fid, "geometry": None, "properties": {}}) def _make_feat_with_geom(fid: str, geom: dict[str, Any]) -> NSPDFeature: return NSPDFeature.from_raw({"id": fid, "geometry": geom, "properties": {"k": "v"}}) def _make_dump( quarter_cad: str = "66:41:0204016", include_zouit: bool = True, include_risks: bool = False, quarter_feat: NSPDFeature | None = None, bbox: tuple[float, float, float, float] | None = (100.0, 200.0, 300.0, 400.0), ) -> QuarterDump: """Фабрика QuarterDump для тестов.""" zouit: dict[str, list[NSPDFeature]] = {} if include_zouit: zouit = { "okn": [_make_feat("ok1"), _make_feat("ok2")], "engineering": [_make_feat("eng1")], "natural": [], "protected": [_make_feat("prot1")], "other": [], } risks: dict[str, list[NSPDFeature]] = {} if include_risks: risks = {"flooding": [_make_feat("fl1")], "landslide": []} return QuarterDump( quarter_cad=quarter_cad, quarter=quarter_feat, parcels=[_make_feat("p1"), _make_feat("p2")], buildings=[_make_feat("b1")], territorial_zones=[_make_feat("tz1")], red_lines=[], engineering_structures=[_make_feat("e1")], zouit=zouit, risks=risks, opportunity={}, layers_fetched=( "search", "parcels", "buildings", "territorial_zones", "red_lines", "engineering_structures", ), bbox_3857=bbox, fetched_at_utc="2026-05-12T03:00:00+00:00", ) # ── _build_features_json tests ──────────────────────────────────────────────── def test_features_json_layers_all_present_with_zouit() -> None: """Если include_zouit=True — все 5 ЗОУИТ-групп присутствуют в features_json.""" dump = _make_dump(include_zouit=True) result = _build_features_json(dump) layers = {item["layer"] for item in result} # Core layers assert "parcels" in layers assert "buildings" in layers assert "territorial_zones" in layers assert "engineering_structures" in layers # ЗОУИТ — все 5 ключей (даже с пустыми списками они не попадут — только # populated items) # okn: 2, engineering: 1, natural: 0, protected: 1, other: 0 assert "zouit_okn" in layers assert "zouit_engineering" in layers assert "zouit_protected" in layers # natural и other — пустые, не должны быть в layers assert "zouit_natural" not in layers assert "zouit_other" not in layers def test_features_json_no_zouit_no_risk_layers() -> None: """Без include_zouit — zouit_* и risk_* ключей нет в features_json.""" dump = _make_dump(include_zouit=False, include_risks=False) result = _build_features_json(dump) layers = {item["layer"] for item in result} for layer_name in layers: assert not layer_name.startswith("zouit_") assert not layer_name.startswith("risk_") def test_features_json_each_feature_has_required_keys() -> None: """Каждый элемент features_json содержит layer, feature_id, geometry, properties.""" dump = _make_dump(include_zouit=True) result = _build_features_json(dump) for item in result: assert "layer" in item assert "feature_id" in item assert "geometry" in item assert "properties" in item def test_build_zouit_count_sums_correctly() -> None: """_build_zouit_count суммирует все ЗОУИТ-группы.""" dump = _make_dump(include_zouit=True) # okn:2 + engineering:1 + natural:0 + protected:1 + other:0 = 4 assert _build_zouit_count(dump) == 4 def test_build_risks_count_empty() -> None: """_build_risks_count при пустом risks dict = 0.""" dump = _make_dump(include_risks=False) assert _build_risks_count(dump) == 0 def test_build_risks_count_with_risks() -> None: """_build_risks_count суммирует risk-группы.""" dump = _make_dump(include_zouit=False, include_risks=True) # flooding:1 + landslide:0 = 1 assert _build_risks_count(dump) == 1 # ── harvest_quarter tests ───────────────────────────────────────────────────── @patch("app.workers.tasks.nspd_sync._upsert_dump") @patch("app.workers.tasks.nspd_sync.NSPDClient") def test_harvest_quarter_success( mock_client_cls: MagicMock, mock_upsert: MagicMock, ) -> None: """Happy path: search_by_quarter → return populated QuarterDump → upsert called.""" dump = _make_dump(include_zouit=True) mock_client = MagicMock() mock_client.search_by_quarter.return_value = dump mock_client_cls.return_value = mock_client result = harvest_quarter( "66:41:0204016", region_code=66, include_zouit=True, include_risks=False, ) assert result["quarter_cad"] == "66:41:0204016" assert result["region_code"] == 66 assert result["total_features"] == dump.total_features assert "harvest_duration_ms" in result assert "error" not in result # Verify _upsert_dump called once with correct positional args # Signature: (quarter_cad, region_code, dump, features_json, duration_ms, harvest_error) assert mock_upsert.call_count == 1 upsert_call = mock_upsert.call_args assert upsert_call[0][0] == "66:41:0204016" # quarter_cad assert upsert_call[0][1] == 66 # region_code assert upsert_call[0][2] is dump # dump object assert upsert_call[0][5] is None # harvest_error=None on success @patch("app.workers.tasks.nspd_sync._upsert_dump") @patch("app.workers.tasks.nspd_sync.NSPDClient") def test_harvest_quarter_empty_quarter( mock_client_cls: MagicMock, mock_upsert: MagicMock, ) -> None: """Quarter не найден в NSPD (quarter=None, bbox=None) — empty harvest, no crash.""" empty_dump = QuarterDump( quarter_cad="66:41:9999999", quarter=None, parcels=[], buildings=[], territorial_zones=[], red_lines=[], engineering_structures=[], zouit={"okn": [], "engineering": [], "natural": [], "protected": [], "other": []}, risks={}, opportunity={}, layers_fetched=("search",), bbox_3857=None, fetched_at_utc="2026-05-12T03:00:00+00:00", ) mock_client = MagicMock() mock_client.search_by_quarter.return_value = empty_dump mock_client_cls.return_value = mock_client result = harvest_quarter("66:41:9999999", region_code=66) # Should succeed with zero counts assert result["quarter_cad"] == "66:41:9999999" assert result["total_features"] == 0 assert "error" not in result # Upsert called with the empty dump (not None) # Signature: (quarter_cad, region_code, dump, features_json, duration_ms, harvest_error) assert mock_upsert.call_count == 1 upsert_args = mock_upsert.call_args[0] assert upsert_args[2] is empty_dump # dump passed assert upsert_args[5] is None # harvest_error=None (valid empty harvest) # features_json should be empty list features_json_arg = upsert_args[3] assert features_json_arg == [] @patch("app.workers.tasks.nspd_sync._upsert_dump") @patch("app.workers.tasks.nspd_sync.NSPDClient") def test_harvest_quarter_waf_retry( mock_client_cls: MagicMock, mock_upsert: MagicMock, ) -> None: """NspdLiteWafError → task re-raises (Celery autoretry_for будет retry). autoretry_for=(NspdLiteWafError,) настроен в декораторе задачи. Тест проверяет что re-raise происходит (task не глотает WAF). """ mock_client = MagicMock() mock_client.search_by_quarter.side_effect = NspdLiteWafError("HTTP 429 WAF") mock_client_cls.return_value = mock_client with pytest.raises(NspdLiteWafError): harvest_quarter("66:41:0204016", region_code=66) # _upsert_dump НЕ должен быть вызван при WAF (не пишем error row, retry сделает попытку) mock_upsert.assert_not_called() @patch("app.workers.tasks.nspd_sync._upsert_dump") @patch("app.workers.tasks.nspd_sync.NSPDClient") def test_harvest_quarter_writes_error_row( mock_client_cls: MagicMock, mock_upsert: MagicMock, ) -> None: """Generic exception → error row inserted с harvest_error set; task НЕ raise.""" mock_client = MagicMock() mock_client.search_by_quarter.side_effect = ConnectionError("timeout") mock_client_cls.return_value = mock_client result = harvest_quarter("66:41:0204016", region_code=66) # Task returns error dict instead of raising assert "error" in result assert "ConnectionError" in result["error"] assert result["quarter_cad"] == "66:41:0204016" # _upsert_dump called with dump=None (error-only row) # Signature: (quarter_cad, region_code, dump, features_json, duration_ms, harvest_error) assert mock_upsert.call_count == 1 upsert_args = mock_upsert.call_args[0] assert upsert_args[2] is None # dump=None assert upsert_args[3] is None # features_json=None assert upsert_args[5] is not None # harvest_error set assert "ConnectionError" in upsert_args[5] # ── harvest_stale_quarters tests ─────────────────────────────────────────────── @patch("app.workers.tasks.nspd_sync.harvest_quarter") @patch("app.workers.tasks.nspd_sync.SessionLocal") def test_harvest_stale_quarters_fanout( mock_session_cls: MagicMock, mock_task: MagicMock, ) -> None: """Stale cads найдены → harvest_quarter.apply_async вызван N раз.""" stale_cads = ["66:41:0000001", "66:41:0000002", "66:41:0000003"] # Mock DB session mock_db = MagicMock() mock_session_cls.return_value = mock_db mock_db.execute.return_value.all.return_value = [(c,) for c in stale_cads] # harvest_quarter.apply_async — проверяем что вызван для каждого cad mock_task.apply_async = MagicMock() result = harvest_stale_quarters(region_code=66, max_age_days=90, batch_size=50) assert result["enqueued"] == 3 assert result["stale_found"] == 3 assert result["region_code"] == 66 assert mock_task.apply_async.call_count == 3 # apply_async(args=[cad, region_code]) — args passed as keyword 'args' called_cads = [c.kwargs["args"][0] for c in mock_task.apply_async.call_args_list] assert called_cads == stale_cads @patch("app.workers.tasks.nspd_sync.harvest_quarter") @patch("app.workers.tasks.nspd_sync.SessionLocal") def test_harvest_stale_quarters_empty( mock_session_cls: MagicMock, mock_task: MagicMock, ) -> None: """Нет stale кварталов → 0 enqueued, apply_async не вызван.""" mock_db = MagicMock() mock_session_cls.return_value = mock_db mock_db.execute.return_value.all.return_value = [] mock_task.apply_async = MagicMock() result = harvest_stale_quarters(region_code=66) assert result["enqueued"] == 0 assert result["stale_found"] == 0 mock_task.apply_async.assert_not_called() # ── features_json layer names tests ────────────────────────────────────────── def test_features_json_risk_layer_names() -> None: """Risk layers кодируются как risk_.""" dump = _make_dump(include_zouit=False, include_risks=True) result = _build_features_json(dump) layers = {item["layer"] for item in result} # flooding:1 item assert "risk_flooding" in layers # landslide:0 items — не должно быть ключа assert "risk_landslide" not in layers def test_features_json_geometry_preserved() -> None: """Geometry (EPSG:3857) сохраняется без трансформации в features_json.""" geom = { "type": "Polygon", "coordinates": [ [[100.0, 200.0], [300.0, 200.0], [300.0, 400.0], [100.0, 400.0], [100.0, 200.0]] ], } dump = QuarterDump( quarter_cad="66:41:0204016", quarter=None, parcels=[_make_feat_with_geom("p1", geom)], buildings=[], territorial_zones=[], red_lines=[], engineering_structures=[], zouit={}, risks={}, opportunity={}, layers_fetched=("search", "parcels"), bbox_3857=(100.0, 200.0, 300.0, 400.0), fetched_at_utc="2026-05-12T00:00:00+00:00", ) result = _build_features_json(dump) assert len(result) == 1 assert result[0]["geometry"] == geom # not transformed assert result[0]["layer"] == "parcels" assert result[0]["feature_id"] == "p1" assert result[0]["properties"] == {"k": "v"} # ── _upsert_dump CAST regression tests (#244) ──────────────────────────────── @patch("app.workers.tasks.nspd_sync.SessionLocal") def test_upsert_dump_null_geom_executes_without_error(mock_session_cls: MagicMock) -> None: """Regression #244: _upsert_dump с geom_json=None не падает на AmbiguousParameter. psycopg3 не может вывести тип голого параметра $N внутри CASE WHEN ... IS NULL. Fix: CAST(:geom_json AS text) IS NULL даёт явный тип — нет AmbiguousParameter. """ mock_db = MagicMock() mock_session_cls.return_value = mock_db # dump=None → error-only path: все geo-параметры None, должен execute без ошибок _upsert_dump( quarter_cad="66:41:0204016", region_code=66, dump=None, features_json=None, duration_ms=100, harvest_error="TestError: simulated", ) mock_db.execute.assert_called_once() mock_db.commit.assert_called_once() @patch("app.workers.tasks.nspd_sync.SessionLocal") def test_upsert_dump_with_geom_executes_without_error(mock_session_cls: MagicMock) -> None: """Regression #244: _upsert_dump с реальным geom_json и bbox не падает. CAST(:geom_json AS text) передаёт строку, CAST(:bbox_xmin AS double precision) — число. Оба branch CASE WHEN типизированы корректно. """ mock_db = MagicMock() mock_session_cls.return_value = mock_db geom = { "type": "Polygon", "coordinates": [ [[100.0, 200.0], [300.0, 200.0], [300.0, 400.0], [100.0, 400.0], [100.0, 200.0]] ], } dump = _make_dump( quarter_cad="66:41:0204016", quarter_feat=_make_feat_with_geom("q1", geom), bbox=(100.0, 200.0, 300.0, 400.0), ) _upsert_dump( quarter_cad="66:41:0204016", region_code=66, dump=dump, features_json=_build_features_json(dump), duration_ms=500, harvest_error=None, ) mock_db.execute.assert_called_once() # Verify params contain properly-typed values (not None for geo fields) call_params = mock_db.execute.call_args[0][1] assert call_params["geom_json"] is not None # JSON string assert call_params["bbox_xmin"] == 100.0 assert call_params["bbox_ymax"] == 400.0 mock_db.commit.assert_called_once()