db1ff5b1fc
- data/sql/99_nspd_entities_denorm.sql: 2 tables (cad_num PK) + 6 indexes (quarter_cad, GIST geom, partial WHERE purpose ILIKE '%многокв%') - nspd_denorm.py: denorm_parcel_feature / denorm_building_feature / denorm_dump - SAVEPOINT per row via with db.begin_nested() - ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(:geom), 3857), 4326) - json.dumps(geom, ensure_ascii=False) for psycopg bind - All CAST(:x AS type) — no ❌:type - nspd_sync.py: inline denorm after _upsert_dump in harvest_quarter (non-fatal) - nspd_denorm_backfill.py: Celery one-shot task (per-quarter commit) - admin_etl.py: POST /api/v1/admin/etl/nspd-denorm-backfill (AdminTokenAuth) - 19 tests: coerce helpers, denorm parcel/building, aggregate counts Foundation for downstream features (МКД neighbors lookup, parcel attrs). Part of #94
322 lines
11 KiB
Python
322 lines
11 KiB
Python
"""Тесты для nspd_denorm.py — coerce helpers + denorm_parcel/building/dump.
|
||
|
||
Не требует реального PostgreSQL — мокает db.execute() и db.begin_nested().
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from typing import Any
|
||
from unittest.mock import MagicMock
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.nspd_denorm import (
|
||
_coerce_int,
|
||
_coerce_numeric,
|
||
denorm_building_feature,
|
||
denorm_dump,
|
||
denorm_parcel_feature,
|
||
)
|
||
|
||
# ── _coerce_int ────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_coerce_int_str_number() -> None:
|
||
assert _coerce_int("5") == 5
|
||
|
||
|
||
def test_coerce_int_int_passthrough() -> None:
|
||
assert _coerce_int(17) == 17
|
||
|
||
|
||
def test_coerce_int_none_returns_none() -> None:
|
||
assert _coerce_int(None) is None
|
||
|
||
|
||
def test_coerce_int_invalid_returns_none() -> None:
|
||
assert _coerce_int("не_число") is None
|
||
|
||
|
||
def test_coerce_int_float_truncates() -> None:
|
||
assert _coerce_int(3.9) == 3
|
||
|
||
|
||
# ── _coerce_numeric ────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_coerce_numeric_float_passthrough() -> None:
|
||
assert _coerce_numeric(12.5) == 12.5
|
||
|
||
|
||
def test_coerce_numeric_str_dot() -> None:
|
||
assert _coerce_numeric("1234567.89") == pytest.approx(1234567.89)
|
||
|
||
|
||
def test_coerce_numeric_comma_decimal() -> None:
|
||
"""Европейский формат с запятой → корректный float."""
|
||
assert _coerce_numeric("1234567,89") == pytest.approx(1234567.89)
|
||
|
||
|
||
def test_coerce_numeric_none_returns_none() -> None:
|
||
assert _coerce_numeric(None) is None
|
||
|
||
|
||
def test_coerce_numeric_invalid_returns_none() -> None:
|
||
assert _coerce_numeric("N/A") is None
|
||
|
||
|
||
# ── Helpers для моков ──────────────────────────────────────────────────────────
|
||
|
||
|
||
def _make_mock_session() -> MagicMock:
|
||
"""Мок Session: begin_nested() — context manager, execute() → MagicMock."""
|
||
db = MagicMock()
|
||
# begin_nested() должен быть context manager
|
||
cm = MagicMock()
|
||
cm.__enter__ = MagicMock(return_value=None)
|
||
cm.__exit__ = MagicMock(return_value=False)
|
||
db.begin_nested.return_value = cm
|
||
return db
|
||
|
||
|
||
def _parcel_feature(
|
||
cad_num: str = "66:41:0204016:10",
|
||
area: Any = "500.0",
|
||
cost_value: Any = "1500000",
|
||
**extra_props: Any,
|
||
) -> dict[str, Any]:
|
||
props: dict[str, Any] = {
|
||
"cad_num": cad_num,
|
||
"area": area,
|
||
"cost_value": cost_value,
|
||
"permitted_use": "Для ИЖС",
|
||
"land_category": "Земли населённых пунктов",
|
||
"address": "г. Екатеринбург",
|
||
**extra_props,
|
||
}
|
||
return {
|
||
"layer": "parcels",
|
||
"feature_id": "123",
|
||
"geometry": {"type": "Polygon", "coordinates": [[[0, 0], [1, 0], [1, 1], [0, 0]]]},
|
||
"properties": props,
|
||
}
|
||
|
||
|
||
def _building_feature(
|
||
cad_num: str = "66:41:0204016:10:1",
|
||
purpose: str = "Многоквартирный дом",
|
||
**extra_props: Any,
|
||
) -> dict[str, Any]:
|
||
props: dict[str, Any] = {
|
||
"cad_num": cad_num,
|
||
"purpose": purpose,
|
||
"floors_above_ground": 9,
|
||
"floors_underground": 1,
|
||
"year_built": 1985,
|
||
"cost_value": "50000000",
|
||
"build_record_area": "3200.0",
|
||
"address": "ул. Ленина 1",
|
||
**extra_props,
|
||
}
|
||
return {
|
||
"layer": "buildings",
|
||
"feature_id": "456",
|
||
"geometry": {"type": "Polygon", "coordinates": [[[0, 0], [1, 0], [1, 1], [0, 0]]]},
|
||
"properties": props,
|
||
}
|
||
|
||
|
||
# ── denorm_parcel_feature ──────────────────────────────────────────────────────
|
||
|
||
|
||
def test_denorm_parcel_feature_inserts() -> None:
|
||
"""Полный feature с cad_num → вызывает db.execute с правильными параметрами."""
|
||
db = _make_mock_session()
|
||
|
||
result = denorm_parcel_feature(
|
||
db,
|
||
feature=_parcel_feature(),
|
||
quarter_cad="66:41:0204016",
|
||
snapshot_date="2026-05-16",
|
||
)
|
||
|
||
assert result is True
|
||
db.begin_nested.assert_called_once()
|
||
db.execute.assert_called_once()
|
||
# Проверяем что параметры содержат нужные ключи
|
||
call_kwargs = db.execute.call_args[0][1] # positional dict
|
||
assert call_kwargs["cad_num"] == "66:41:0204016:10"
|
||
assert call_kwargs["quarter_cad"] == "66:41:0204016"
|
||
assert call_kwargs["permitted_use"] == "Для ИЖС"
|
||
assert call_kwargs["land_category"] == "Земли населённых пунктов"
|
||
assert call_kwargs["area_sqm"] == 500.0
|
||
assert call_kwargs["cost_value"] == 1500000.0
|
||
assert call_kwargs["cost_per_m2"] == pytest.approx(3000.0)
|
||
assert call_kwargs["snapshot_date"] == "2026-05-16"
|
||
|
||
|
||
def test_denorm_parcel_no_cad_num_skipped() -> None:
|
||
"""Feature без cad_num → возвращает False, db не вызывается."""
|
||
db = _make_mock_session()
|
||
feature: dict[str, Any] = {
|
||
"layer": "parcels",
|
||
"feature_id": None,
|
||
"geometry": None,
|
||
"properties": {"area": "100"},
|
||
}
|
||
result = denorm_parcel_feature(
|
||
db, feature=feature, quarter_cad="66:41:0204016", snapshot_date="2026-05-16"
|
||
)
|
||
assert result is False
|
||
db.execute.assert_not_called()
|
||
|
||
|
||
def test_denorm_parcel_zero_area_cost_per_m2_none() -> None:
|
||
"""area=0 → cost_per_m2 = None (нет деления на ноль)."""
|
||
db = _make_mock_session()
|
||
result = denorm_parcel_feature(
|
||
db,
|
||
feature=_parcel_feature(area="0", cost_value="1000"),
|
||
quarter_cad="66:41:0204016",
|
||
snapshot_date="2026-05-16",
|
||
)
|
||
assert result is True
|
||
call_kwargs = db.execute.call_args[0][1]
|
||
assert call_kwargs["cost_per_m2"] is None
|
||
|
||
|
||
def test_denorm_parcel_null_geometry() -> None:
|
||
"""feature без geometry → geom_json=None, INSERT всё равно вызывается."""
|
||
db = _make_mock_session()
|
||
feature = _parcel_feature()
|
||
feature["geometry"] = None
|
||
|
||
result = denorm_parcel_feature(
|
||
db, feature=feature, quarter_cad="66:41:0204016", snapshot_date="2026-05-16"
|
||
)
|
||
assert result is True
|
||
call_kwargs = db.execute.call_args[0][1]
|
||
assert call_kwargs["geom_json"] is None
|
||
|
||
|
||
def test_denorm_parcel_db_exception_returns_false() -> None:
|
||
"""DB execute raises → возвращает False (не propagate)."""
|
||
db = _make_mock_session()
|
||
db.execute.side_effect = Exception("DB constraint violation")
|
||
|
||
result = denorm_parcel_feature(
|
||
db,
|
||
feature=_parcel_feature(),
|
||
quarter_cad="66:41:0204016",
|
||
snapshot_date="2026-05-16",
|
||
)
|
||
assert result is False
|
||
|
||
|
||
# ── denorm_building_feature ────────────────────────────────────────────────────
|
||
|
||
|
||
def test_denorm_building_feature_inserts() -> None:
|
||
"""Полный building feature → INSERT с правильными параметрами."""
|
||
db = _make_mock_session()
|
||
|
||
result = denorm_building_feature(
|
||
db,
|
||
feature=_building_feature(),
|
||
quarter_cad="66:41:0204016",
|
||
snapshot_date="2026-05-16",
|
||
)
|
||
|
||
assert result is True
|
||
db.execute.assert_called_once()
|
||
call_kwargs = db.execute.call_args[0][1]
|
||
assert call_kwargs["cad_num"] == "66:41:0204016:10:1"
|
||
assert call_kwargs["purpose"] == "Многоквартирный дом"
|
||
assert call_kwargs["floors"] == 9
|
||
assert call_kwargs["floors_underground"] == 1
|
||
assert call_kwargs["year_built"] == 1985
|
||
assert call_kwargs["cost_value"] == 50000000.0
|
||
assert call_kwargs["build_record_area"] == 3200.0
|
||
|
||
|
||
def test_denorm_building_no_cad_num_skipped() -> None:
|
||
"""Building feature без cad_num → False."""
|
||
db = _make_mock_session()
|
||
feature: dict[str, Any] = {
|
||
"layer": "buildings",
|
||
"feature_id": None,
|
||
"geometry": None,
|
||
"properties": {"purpose": "Нежилое здание"},
|
||
}
|
||
result = denorm_building_feature(
|
||
db, feature=feature, quarter_cad="66:41:0204016", snapshot_date="2026-05-16"
|
||
)
|
||
assert result is False
|
||
db.execute.assert_not_called()
|
||
|
||
|
||
def test_denorm_building_str_floors_coerced() -> None:
|
||
"""floors_above_ground строкой → корректно парсится в int."""
|
||
db = _make_mock_session()
|
||
result = denorm_building_feature(
|
||
db,
|
||
feature=_building_feature(floors_above_ground="12"),
|
||
quarter_cad="66:41:0204016",
|
||
snapshot_date="2026-05-16",
|
||
)
|
||
assert result is True
|
||
call_kwargs = db.execute.call_args[0][1]
|
||
assert call_kwargs["floors"] == 12
|
||
|
||
|
||
# ── denorm_dump ────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_denorm_dump_aggregates() -> None:
|
||
"""3 parcels + 2 buildings + 1 unknown layer → правильные счётчики."""
|
||
db = _make_mock_session()
|
||
|
||
features: list[dict[str, Any]] = [
|
||
_parcel_feature("66:41:0101001:1"),
|
||
_parcel_feature("66:41:0101001:2"),
|
||
_parcel_feature("66:41:0101001:3"),
|
||
_building_feature("66:41:0101001:1:1"),
|
||
_building_feature("66:41:0101001:1:2"),
|
||
{
|
||
"layer": "territorial_zones",
|
||
"feature_id": "tz-1",
|
||
"geometry": None,
|
||
"properties": {"type_zone": "Ж-1"},
|
||
},
|
||
]
|
||
|
||
counts = denorm_dump(db, quarter_cad="66:41:0101001", features=features)
|
||
|
||
assert counts["parcels"] == 3
|
||
assert counts["buildings"] == 2
|
||
# territorial_zones не считается как ошибка — просто пропускается
|
||
assert counts["errors"] == 0
|
||
db.commit.assert_called_once()
|
||
|
||
|
||
def test_denorm_dump_empty_features() -> None:
|
||
"""Пустой список features → нули, commit всё равно вызывается."""
|
||
db = _make_mock_session()
|
||
counts = denorm_dump(db, quarter_cad="66:41:0101001", features=[])
|
||
|
||
assert counts == {"parcels": 0, "buildings": 0, "errors": 0}
|
||
db.commit.assert_called_once()
|
||
|
||
|
||
def test_denorm_dump_no_cad_num_counted_as_error() -> None:
|
||
"""Parcel без cad_num → denorm_parcel_feature returns False → errors += 1."""
|
||
db = _make_mock_session()
|
||
feature: dict[str, Any] = {
|
||
"layer": "parcels",
|
||
"feature_id": None,
|
||
"geometry": None,
|
||
"properties": {},
|
||
}
|
||
counts = denorm_dump(db, quarter_cad="66:41:0101001", features=[feature])
|
||
assert counts["parcels"] == 0
|
||
assert counts["errors"] == 1
|