5da66bbc93
* feat(cadastre): bulk_harvest_quarter Celery task + grid-walker + saga state (#168 PR3/5) Add bulk cadastre harvest pipeline: - services/cadastre/bulk_harvest.py: async harvest_quarter() orchestrator (4 phases) + 7 upsert helpers (parcels/buildings/constructions/oncs/enks/zouit/quarter_stats) using CAST(:x AS jsonb) pattern, begin_nested() SAVEPOINT per grid-walk upsert - services/cadastre/grid_geometry.py: quarter_bbox_3857 (PostGIS ST_Extent) + generate_grid_click_points (15x15 = 225 sub-bbox grid) - workers/tasks/scrape_cadastre.py: bulk_harvest_quarter_task (acks_late=True, dont_autoretry_for NspdBulkWafError), enqueue_cadastre_harvest, cleanup_zombies - api/v1/admin_cadastre.py: 5 endpoints behind AdminTokenAuth — create/list/get/cancel/resume - Tests: 13 service unit + 8 API tests * fixup(cadastre): drop importorskip (PR2 merged) — imports now top-level (#168 PR3) * fixup(cadastre): tile_size unified param + register slow marker (#168 PR3) Blocker #1: rename tile_width/tile_height → tile_size in generate_grid_click_points. Callers in bulk_harvest.py and test_cadastre_bulk.py used tile_size; def had tile_width+tile_height → TypeError at runtime. Blocker #2 (10 failures in test_admin_cadastre.py) was side-effect of #1: TypeError at import chain (app.main → admin_cadastre → bulk_harvest) broke FastAPI app load → dependency_overrides AttributeError. Now resolved. Also register `slow` pytest marker in pyproject.toml to suppress PytestUnknownMarkWarning. * fixup(cadastre): use importlib.util.find_spec instead of import (#168 PR3) Root cause Blocker #2: `import app.workers.tasks.scrape_cadastre` at module top-level REBOUND `app` from FastAPI instance (line 14) to the Python package. All subsequent `app.dependency_overrides[get_db] = ...` failed with AttributeError because `app` was now the namespace module. Fix: probe module availability with importlib.util.find_spec — does not import or rebind names. FastAPI `app` instance stays intact. All 28 cadastre tests pass locally. --------- Co-authored-by: lekss361 <claudestars@proton.me>
490 lines
17 KiB
Python
490 lines
17 KiB
Python
"""Тесты для bulk_harvest.py и grid_geometry.py.
|
||
|
||
Использует mock DB (через unittest.mock) и mock NSPDBulkClient.
|
||
Не требует реального PostgreSQL или NSPD сети.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from typing import Any
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from app.schemas.nspd_bulk import NSPDBulkFeature, QuarterSnapshot
|
||
from app.services.cadastre.bulk_harvest import (
|
||
_derive_quarter_cad,
|
||
_safe_int,
|
||
_safe_numeric,
|
||
upsert_features,
|
||
)
|
||
from app.services.cadastre.grid_geometry import generate_grid_click_points
|
||
|
||
# ── Фикстуры ─────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _make_parcel_feature(cad_num: str = "66:41:0303161:1") -> NSPDBulkFeature:
|
||
"""Создать NSPDBulkFeature для ЗУ (category_id=36368)."""
|
||
return NSPDBulkFeature.model_validate(
|
||
{
|
||
"id": 1,
|
||
"type": "Feature",
|
||
"geometry": {
|
||
"type": "Polygon",
|
||
"coordinates": [
|
||
[
|
||
[6090000, 7590000],
|
||
[6090100, 7590000],
|
||
[6090100, 7590100],
|
||
[6090000, 7590100],
|
||
[6090000, 7590000],
|
||
]
|
||
],
|
||
},
|
||
"properties": {
|
||
"categoryId": 36368,
|
||
"options": {
|
||
"cad_num": cad_num,
|
||
"quarter_cad_number": "66:41:0303161",
|
||
"land_record_area": 1234.56,
|
||
"cost_value": 5000000.0,
|
||
"readable_address": "г Екатеринбург, ул Тестовая",
|
||
"status": "Учтённый",
|
||
"ownership_type": "частная",
|
||
},
|
||
},
|
||
}
|
||
)
|
||
|
||
|
||
def _make_building_feature(cad_num: str = "66:41:0303161:1:1") -> NSPDBulkFeature:
|
||
"""Создать NSPDBulkFeature для здания (category_id=36369)."""
|
||
return NSPDBulkFeature.model_validate(
|
||
{
|
||
"id": 2,
|
||
"type": "Feature",
|
||
"geometry": {
|
||
"type": "Polygon",
|
||
"coordinates": [
|
||
[
|
||
[6090000, 7590000],
|
||
[6090050, 7590000],
|
||
[6090050, 7590050],
|
||
[6090000, 7590050],
|
||
[6090000, 7590000],
|
||
]
|
||
],
|
||
},
|
||
"properties": {
|
||
"categoryId": 36369,
|
||
"options": {
|
||
"cad_num": cad_num,
|
||
"quarter_cad_number": "66:41:0303161",
|
||
"purpose": "Жилое",
|
||
"floors": "9",
|
||
"area": 5000.0,
|
||
"cost_value": 20000000.0,
|
||
"readable_address": "г Екатеринбург, ул Тестовая, д 1",
|
||
},
|
||
},
|
||
}
|
||
)
|
||
|
||
|
||
def _make_zouit_feature(reg_numb: str = "66-0.1-1.1-1234") -> NSPDBulkFeature:
|
||
"""Создать NSPDBulkFeature для ЗОУИТ (category_id=36940)."""
|
||
return NSPDBulkFeature.model_validate(
|
||
{
|
||
"id": 3,
|
||
"type": "Feature",
|
||
"geometry": {
|
||
"type": "MultiPolygon",
|
||
"coordinates": [
|
||
[
|
||
[
|
||
[6090000, 7590000],
|
||
[6091000, 7590000],
|
||
[6091000, 7591000],
|
||
[6090000, 7591000],
|
||
[6090000, 7590000],
|
||
]
|
||
]
|
||
],
|
||
},
|
||
"properties": {
|
||
"categoryId": 36940,
|
||
"categoryName": "ЗОУИТ",
|
||
"options": {
|
||
"reg_numb_border": reg_numb,
|
||
"type_zone": "Охранная зона ЛЭП",
|
||
"registration_date": "2020-01-01",
|
||
},
|
||
},
|
||
}
|
||
)
|
||
|
||
|
||
# ── Вспомогательные функции ───────────────────────────────────────────────────
|
||
|
||
|
||
def test_derive_quarter_cad_4_segments() -> None:
|
||
assert _derive_quarter_cad("66:41:0303161:1") == "66:41:0303161"
|
||
|
||
|
||
def test_derive_quarter_cad_5_segments() -> None:
|
||
assert _derive_quarter_cad("66:41:0303161:1:1") == "66:41:0303161"
|
||
|
||
|
||
def test_derive_quarter_cad_3_segments_unchanged() -> None:
|
||
assert _derive_quarter_cad("66:41:0303161") == "66:41:0303161"
|
||
|
||
|
||
def test_safe_numeric_valid() -> None:
|
||
assert _safe_numeric("1234.56") == pytest.approx(1234.56)
|
||
assert _safe_numeric(100) == pytest.approx(100.0)
|
||
|
||
|
||
def test_safe_numeric_none_returns_none() -> None:
|
||
assert _safe_numeric(None) is None
|
||
|
||
|
||
def test_safe_numeric_invalid_returns_none() -> None:
|
||
assert _safe_numeric("не число") is None
|
||
|
||
|
||
def test_safe_int_valid() -> None:
|
||
assert _safe_int("9") == 9
|
||
assert _safe_int(5.7) == 5
|
||
|
||
|
||
def test_safe_int_none_returns_none() -> None:
|
||
assert _safe_int(None) is None
|
||
|
||
|
||
# ── Grid geometry ────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_generate_grid_click_points_count() -> None:
|
||
"""15×15 grid → 225 точек."""
|
||
bbox = (6090000.0, 7590000.0, 6090750.0, 7590750.0)
|
||
points = generate_grid_click_points(bbox, grid_size=15)
|
||
assert len(points) == 225
|
||
|
||
|
||
def test_generate_grid_click_points_bbox_coverage() -> None:
|
||
"""Все sub-bbox покрывают исходный bbox без перекрытий."""
|
||
bbox = (0.0, 0.0, 1500.0, 1500.0)
|
||
points = generate_grid_click_points(bbox, grid_size=3)
|
||
# 3×3 = 9 ячеек
|
||
assert len(points) == 9
|
||
# Первая ячейка начинается с 0,0
|
||
first_bbox, first_click = points[0]
|
||
assert first_bbox[0] == pytest.approx(0.0)
|
||
assert first_bbox[1] == pytest.approx(0.0)
|
||
# click_xy в центре тайла (256, 256) для tile_size=512
|
||
assert first_click == (256, 256)
|
||
|
||
|
||
def test_generate_grid_click_points_custom_grid() -> None:
|
||
bbox = (0.0, 0.0, 1000.0, 1000.0)
|
||
points = generate_grid_click_points(bbox, grid_size=5, tile_size=256)
|
||
assert len(points) == 25
|
||
# Каждая ячейка имеет правильный размер 200м×200м
|
||
cell_bbox, _ = points[0]
|
||
assert cell_bbox[2] - cell_bbox[0] == pytest.approx(200.0)
|
||
assert cell_bbox[3] - cell_bbox[1] == pytest.approx(200.0)
|
||
|
||
|
||
# ── upsert_features routing ──────────────────────────────────────────────────
|
||
|
||
|
||
def test_upsert_features_routes_parcels_and_buildings() -> None:
|
||
"""upsert_features должен вызывать upsert_parcel и upsert_building."""
|
||
parcel = _make_parcel_feature()
|
||
building = _make_building_feature()
|
||
|
||
db = MagicMock()
|
||
db.execute = MagicMock(return_value=MagicMock())
|
||
db.begin_nested = MagicMock(return_value=MagicMock(__enter__=MagicMock(), __exit__=MagicMock()))
|
||
|
||
with (
|
||
patch("app.services.cadastre.bulk_harvest.upsert_parcel") as mock_parcel,
|
||
patch("app.services.cadastre.bulk_harvest.upsert_building") as mock_building,
|
||
):
|
||
counts = upsert_features(db, [parcel, building], source="search")
|
||
|
||
mock_parcel.assert_called_once_with(db, parcel, source="search")
|
||
mock_building.assert_called_once_with(db, building, source="search")
|
||
assert counts["parcels"] == 1
|
||
assert counts["buildings"] == 1
|
||
|
||
|
||
def test_upsert_features_routes_zouit() -> None:
|
||
"""ЗОУИТ (36940) маршрутируется в upsert_zouit."""
|
||
zouit = _make_zouit_feature()
|
||
db = MagicMock()
|
||
|
||
with patch("app.services.cadastre.bulk_harvest.upsert_zouit") as mock_zouit:
|
||
counts = upsert_features(db, [zouit], source="search")
|
||
|
||
mock_zouit.assert_called_once()
|
||
assert counts["zouit"] == 1
|
||
|
||
|
||
def test_upsert_features_unknown_category_counts_as_skipped() -> None:
|
||
"""Неизвестный category_id → skipped, не raise."""
|
||
unknown_feature = NSPDBulkFeature.model_validate(
|
||
{
|
||
"type": "Feature",
|
||
"geometry": None,
|
||
"properties": {"categoryId": 99999, "options": {}},
|
||
}
|
||
)
|
||
db = MagicMock()
|
||
counts = upsert_features(db, [unknown_feature], source="search")
|
||
assert counts["skipped"] == 1
|
||
|
||
|
||
def test_upsert_features_quarter_stats_not_counted() -> None:
|
||
"""category_id=36381 (quarter stats) — не считается ни в одну таблицу."""
|
||
stats_feature = NSPDBulkFeature.model_validate(
|
||
{
|
||
"type": "Feature",
|
||
"geometry": None,
|
||
"properties": {"categoryId": 36381, "options": {"cnt_land": 5}},
|
||
}
|
||
)
|
||
db = MagicMock()
|
||
counts = upsert_features(db, [stats_feature], source="search")
|
||
# Все счётчики должны быть 0, skipped тоже (36381 — особый случай)
|
||
assert counts["parcels"] == 0
|
||
assert counts["buildings"] == 0
|
||
assert counts["skipped"] == 0
|
||
|
||
|
||
# ── upsert_parcel SQL params ─────────────────────────────────────────────────
|
||
|
||
|
||
def test_upsert_parcel_uses_cast_not_double_colon() -> None:
|
||
"""Проверить что SQL использует CAST(:x AS type) а не :x::type (SQLAlchemy ловушка).
|
||
|
||
Используем inspect самого SQL через db.execute calls.
|
||
"""
|
||
from app.services.cadastre.bulk_harvest import upsert_parcel
|
||
|
||
db = MagicMock()
|
||
executed_sqls: list[str] = []
|
||
|
||
def capture_execute(stmt: Any, params: Any = None) -> MagicMock:
|
||
executed_sqls.append(str(stmt))
|
||
return MagicMock()
|
||
|
||
db.execute = capture_execute
|
||
|
||
feature = _make_parcel_feature()
|
||
upsert_parcel(db, feature, source="search")
|
||
|
||
assert len(executed_sqls) == 1
|
||
sql = executed_sqls[0]
|
||
# Проверяем что нет `:x::type` паттерна (только CAST(:x AS ...))
|
||
assert "::jsonb" not in sql, "SQL содержит ::jsonb — нужно CAST(:x AS jsonb)"
|
||
assert "::date" not in sql, "SQL содержит ::date — нужно CAST(:x AS date)"
|
||
assert "CAST(:raw_props AS jsonb)" in sql
|
||
|
||
|
||
# ── harvest_quarter integration (async, mocked) ───────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_harvest_quarter_phase_done_early_return() -> None:
|
||
"""Если phase_state='done' — harvest_quarter возвращается ранним выходом."""
|
||
from app.services.cadastre.bulk_harvest import harvest_quarter
|
||
|
||
db = MagicMock()
|
||
db.execute = MagicMock(
|
||
return_value=MagicMock(
|
||
mappings=MagicMock(
|
||
return_value=MagicMock(
|
||
first=MagicMock(return_value={"phase_state": {"phase": "done"}})
|
||
)
|
||
)
|
||
)
|
||
)
|
||
|
||
client = AsyncMock()
|
||
progress_calls: list[Any] = []
|
||
|
||
await harvest_quarter(
|
||
db=db,
|
||
client=client,
|
||
quarter="66:41:0303161",
|
||
job_id=1,
|
||
update_progress=lambda s: progress_calls.append(s),
|
||
)
|
||
|
||
# client.search_by_quarter не должен вызываться
|
||
client.search_by_quarter.assert_not_called()
|
||
# Нет progress calls
|
||
assert len(progress_calls) == 0
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_harvest_quarter_calls_upsert_features() -> None:
|
||
"""harvest_quarter Phase 1: search_by_quarter → upsert_features."""
|
||
from app.services.cadastre.bulk_harvest import harvest_quarter
|
||
|
||
parcel = _make_parcel_feature()
|
||
snapshot = QuarterSnapshot(
|
||
quarter="66:41:0303161",
|
||
fetched_at="2026-05-15T10:00:00+00:00",
|
||
features=[parcel],
|
||
meta_counts={36368: 5, 36369: 3}, # нет overflow
|
||
)
|
||
|
||
db = MagicMock()
|
||
# phase_state = None → начинаем с нуля
|
||
db.execute = MagicMock(
|
||
return_value=MagicMock(
|
||
mappings=MagicMock(
|
||
return_value=MagicMock(first=MagicMock(return_value={"phase_state": None}))
|
||
)
|
||
)
|
||
)
|
||
db.commit = MagicMock()
|
||
|
||
client = AsyncMock()
|
||
client.search_by_quarter = AsyncMock(return_value=snapshot)
|
||
|
||
progress_states: list[dict[str, Any]] = []
|
||
|
||
with patch("app.services.cadastre.bulk_harvest.upsert_features") as mock_upsert:
|
||
mock_upsert.return_value = {
|
||
"parcels": 1,
|
||
"buildings": 0,
|
||
"constructions": 0,
|
||
"oncs": 0,
|
||
"enks": 0,
|
||
"zouit": 0,
|
||
"skipped": 0,
|
||
}
|
||
result = await harvest_quarter(
|
||
db=db,
|
||
client=client,
|
||
quarter="66:41:0303161",
|
||
job_id=1,
|
||
update_progress=lambda s: progress_states.append(s),
|
||
)
|
||
|
||
client.search_by_quarter.assert_awaited_once_with("66:41:0303161")
|
||
mock_upsert.assert_called()
|
||
assert result.parcels_upserted == 1
|
||
|
||
# Проверить что последний progress = done
|
||
assert progress_states[-1]["phase"] == "done"
|
||
|
||
|
||
# ── Dedup тест для grid-walk ─────────────────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_grid_walk_dedup_same_cad_num() -> None:
|
||
"""Если wms_feature_info возвращает одинаковый cad_num дважды — 1 upsert."""
|
||
from app.services.cadastre.bulk_harvest import _grid_walk_category
|
||
|
||
parcel = _make_parcel_feature("66:41:0303161:1")
|
||
# Одинаковый feature для каждой grid-точки
|
||
|
||
db = MagicMock()
|
||
db.execute = MagicMock(
|
||
return_value=MagicMock(
|
||
mappings=MagicMock(
|
||
return_value=MagicMock(
|
||
first=MagicMock(
|
||
return_value={
|
||
"xmin": 6090000.0,
|
||
"ymin": 7590000.0,
|
||
"xmax": 6090750.0,
|
||
"ymax": 7590750.0,
|
||
}
|
||
)
|
||
)
|
||
)
|
||
)
|
||
)
|
||
db.begin_nested = MagicMock(
|
||
return_value=MagicMock(__enter__=MagicMock(return_value=None), __exit__=MagicMock())
|
||
)
|
||
db.commit = MagicMock()
|
||
|
||
client = AsyncMock()
|
||
# Каждый wms_feature_info вызов возвращает один и тот же парцел
|
||
client.wms_feature_info = AsyncMock(return_value=[parcel])
|
||
|
||
with patch("app.services.cadastre.bulk_harvest.quarter_bbox_3857") as mock_bbox:
|
||
mock_bbox.return_value = (6090000.0, 7590000.0, 6090750.0, 7590750.0)
|
||
with patch("app.services.cadastre.bulk_harvest.upsert_features") as mock_upsert:
|
||
mock_upsert.return_value = {
|
||
"parcels": 1,
|
||
"buildings": 0,
|
||
"constructions": 0,
|
||
"oncs": 0,
|
||
"enks": 0,
|
||
"zouit": 0,
|
||
"skipped": 0,
|
||
}
|
||
|
||
# Используем grid_size=2 для скорости теста (4 ячейки)
|
||
upserted, requests = await _grid_walk_category(
|
||
db=db,
|
||
client=client,
|
||
quarter="66:41:0303161",
|
||
layer_id=36368,
|
||
grid_size=2,
|
||
)
|
||
|
||
# Несмотря на 4 запроса, upsert_features вызывается 1 раз (дедуп по cad_num)
|
||
assert mock_upsert.call_count == 1
|
||
assert upserted == 1
|
||
assert requests == 4 # 2×2 grid
|
||
|
||
|
||
# ── ЗОУИТ dedup по (reg_numb_border, category_id) ───────────────────────────
|
||
|
||
|
||
def test_zouit_dedup_different_category_ids() -> None:
|
||
"""Один reg_numb_border с разными category_id → 2 записи в upsert."""
|
||
zouit_1 = NSPDBulkFeature.model_validate(
|
||
{
|
||
"type": "Feature",
|
||
"geometry": None,
|
||
"properties": {
|
||
"categoryId": 36940,
|
||
"categoryName": "ЗОУИТ 1",
|
||
"options": {"reg_numb_border": "66-0.1-1.1-1234", "type_zone": "тип А"},
|
||
},
|
||
}
|
||
)
|
||
zouit_2 = NSPDBulkFeature.model_validate(
|
||
{
|
||
"type": "Feature",
|
||
"geometry": None,
|
||
"properties": {
|
||
"categoryId": 469039,
|
||
"categoryName": "ЗОУИТ 2",
|
||
"options": {"reg_numb_border": "66-0.1-1.1-1234", "type_zone": "тип Б"},
|
||
},
|
||
}
|
||
)
|
||
|
||
db = MagicMock()
|
||
calls: list[Any] = []
|
||
db.execute = MagicMock(
|
||
side_effect=lambda stmt, params=None: calls.append(params) or MagicMock()
|
||
)
|
||
|
||
with patch("app.services.cadastre.bulk_harvest.upsert_zouit") as mock_zouit:
|
||
counts = upsert_features(db, [zouit_1, zouit_2], source="search")
|
||
|
||
# Обе записи маршрутируются в upsert_zouit
|
||
assert mock_zouit.call_count == 2
|
||
assert counts["zouit"] == 2
|