db1ff5b1fc
- data/sql/99_nspd_entities_denorm.sql: 2 tables (cad_num PK) + 6 indexes (quarter_cad, GIST geom, partial WHERE purpose ILIKE '%многокв%') - nspd_denorm.py: denorm_parcel_feature / denorm_building_feature / denorm_dump - SAVEPOINT per row via with db.begin_nested() - ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(:geom), 3857), 4326) - json.dumps(geom, ensure_ascii=False) for psycopg bind - All CAST(:x AS type) — no ❌:type - nspd_sync.py: inline denorm after _upsert_dump in harvest_quarter (non-fatal) - nspd_denorm_backfill.py: Celery one-shot task (per-quarter commit) - admin_etl.py: POST /api/v1/admin/etl/nspd-denorm-backfill (AdminTokenAuth) - 19 tests: coerce helpers, denorm parcel/building, aggregate counts Foundation for downstream features (МКД neighbors lookup, parcel attrs). Part of #94
290 lines
11 KiB
Python
290 lines
11 KiB
Python
"""Denormalize nspd_quarter_dumps.features_json → nspd_parcels / nspd_buildings.
|
||
|
||
Используется:
|
||
- Inline в harvest_quarter task после UPSERT dump'а
|
||
- Backfill task для existing dumps (backfill_all_dumps)
|
||
|
||
Property mapping (NSPD WMS GetFeatureInfo responses):
|
||
Parcels (layer="parcels", layer id 36048):
|
||
cad_num — кадастровый номер ЗУ
|
||
permitted_use — ВРИ
|
||
land_category — категория земель
|
||
cost_value — кадастровая стоимость, руб.
|
||
area — площадь, м²
|
||
address — адрес
|
||
|
||
Buildings (layer="buildings", layer id 36049):
|
||
cad_num — кадастровый номер ОКС
|
||
purpose — назначение ("Многоквартирный дом" / "Нежилое здание" / ...)
|
||
floors_above_ground — надземных этажей
|
||
floors_underground — подземных этажей
|
||
year_built — год постройки
|
||
cost_value — кадастровая стоимость, руб.
|
||
build_record_area — площадь по ГКН, м²
|
||
address — адрес
|
||
|
||
Геометрия в features_json хранится в EPSG:3857 (как возвращает NSPD WMS) —
|
||
трансформация 3857→4326 делается в SQL через ST_Transform.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import datetime
|
||
import json
|
||
import logging
|
||
from typing import Any
|
||
|
||
from sqlalchemy import text
|
||
from sqlalchemy.orm import Session
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# ── Type coercions ─────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _coerce_int(v: Any) -> int | None:
|
||
"""NSPD properties могут быть str / int / None."""
|
||
if v is None:
|
||
return None
|
||
try:
|
||
return int(v)
|
||
except (ValueError, TypeError):
|
||
return None
|
||
|
||
|
||
def _coerce_numeric(v: Any) -> float | None:
|
||
"""NSPD properties могут быть str с запятой (европейский формат) или None."""
|
||
if v is None:
|
||
return None
|
||
try:
|
||
return float(str(v).replace(",", ".").strip())
|
||
except (ValueError, TypeError):
|
||
return None
|
||
|
||
|
||
# ── Parcel UPSERT ──────────────────────────────────────────────────────────────
|
||
|
||
_PARCEL_UPSERT_SQL = text(
|
||
"""
|
||
INSERT INTO nspd_parcels (
|
||
cad_num, quarter_cad, permitted_use, land_category,
|
||
cost_value, cost_per_m2, area_sqm, address, geom, snapshot_date
|
||
) VALUES (
|
||
CAST(:cad_num AS text),
|
||
CAST(:quarter_cad AS text),
|
||
CAST(:permitted_use AS text),
|
||
CAST(:land_category AS text),
|
||
CAST(:cost_value AS numeric),
|
||
CAST(:cost_per_m2 AS numeric),
|
||
CAST(:area_sqm AS numeric),
|
||
CAST(:address AS text),
|
||
CASE WHEN :geom_json IS NULL THEN NULL
|
||
ELSE ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(:geom_json), 3857), 4326)
|
||
END,
|
||
CAST(:snapshot_date AS date)
|
||
)
|
||
ON CONFLICT (cad_num) DO UPDATE SET
|
||
quarter_cad = EXCLUDED.quarter_cad,
|
||
permitted_use = EXCLUDED.permitted_use,
|
||
land_category = EXCLUDED.land_category,
|
||
cost_value = EXCLUDED.cost_value,
|
||
cost_per_m2 = EXCLUDED.cost_per_m2,
|
||
area_sqm = EXCLUDED.area_sqm,
|
||
address = EXCLUDED.address,
|
||
geom = EXCLUDED.geom,
|
||
snapshot_date = EXCLUDED.snapshot_date,
|
||
updated_at = NOW()
|
||
"""
|
||
)
|
||
|
||
|
||
def denorm_parcel_feature(
|
||
db: Session,
|
||
*,
|
||
feature: dict[str, Any],
|
||
quarter_cad: str,
|
||
snapshot_date: str,
|
||
) -> bool:
|
||
"""UPSERT one parcel feature → nspd_parcels.
|
||
|
||
Returns True если строка вставлена/обновлена, False при пропуске (нет cad_num).
|
||
"""
|
||
props = feature.get("properties") or {}
|
||
cad_num = props.get("cad_num") or props.get("cadastral_number")
|
||
if not cad_num:
|
||
return False
|
||
|
||
area = _coerce_numeric(props.get("area"))
|
||
cost_value = _coerce_numeric(props.get("cost_value"))
|
||
cost_per_m2: float | None = None
|
||
if cost_value is not None and area is not None and area > 0:
|
||
cost_per_m2 = cost_value / area
|
||
|
||
geom = feature.get("geometry")
|
||
geom_json: str | None = json.dumps(geom, ensure_ascii=False) if geom else None
|
||
|
||
try:
|
||
with db.begin_nested():
|
||
db.execute(
|
||
_PARCEL_UPSERT_SQL,
|
||
{
|
||
"cad_num": cad_num,
|
||
"quarter_cad": quarter_cad,
|
||
"permitted_use": props.get("permitted_use"),
|
||
"land_category": props.get("land_category"),
|
||
"cost_value": cost_value,
|
||
"cost_per_m2": cost_per_m2,
|
||
"area_sqm": area,
|
||
"address": props.get("address"),
|
||
"geom_json": geom_json,
|
||
"snapshot_date": snapshot_date,
|
||
},
|
||
)
|
||
return True
|
||
except Exception as e:
|
||
logger.warning("denorm parcel %s failed: %s", cad_num, e)
|
||
return False
|
||
|
||
|
||
# ── Building UPSERT ────────────────────────────────────────────────────────────
|
||
|
||
_BUILDING_UPSERT_SQL = text(
|
||
"""
|
||
INSERT INTO nspd_buildings (
|
||
cad_num, quarter_cad, purpose, floors, floors_underground,
|
||
year_built, cost_value, build_record_area, address, geom, snapshot_date
|
||
) VALUES (
|
||
CAST(:cad_num AS text),
|
||
CAST(:quarter_cad AS text),
|
||
CAST(:purpose AS text),
|
||
CAST(:floors AS integer),
|
||
CAST(:floors_underground AS integer),
|
||
CAST(:year_built AS integer),
|
||
CAST(:cost_value AS numeric),
|
||
CAST(:build_record_area AS numeric),
|
||
CAST(:address AS text),
|
||
CASE WHEN :geom_json IS NULL THEN NULL
|
||
ELSE ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(:geom_json), 3857), 4326)
|
||
END,
|
||
CAST(:snapshot_date AS date)
|
||
)
|
||
ON CONFLICT (cad_num) DO UPDATE SET
|
||
quarter_cad = EXCLUDED.quarter_cad,
|
||
purpose = EXCLUDED.purpose,
|
||
floors = EXCLUDED.floors,
|
||
floors_underground = EXCLUDED.floors_underground,
|
||
year_built = EXCLUDED.year_built,
|
||
cost_value = EXCLUDED.cost_value,
|
||
build_record_area = EXCLUDED.build_record_area,
|
||
address = EXCLUDED.address,
|
||
geom = EXCLUDED.geom,
|
||
snapshot_date = EXCLUDED.snapshot_date,
|
||
updated_at = NOW()
|
||
"""
|
||
)
|
||
|
||
|
||
def denorm_building_feature(
|
||
db: Session,
|
||
*,
|
||
feature: dict[str, Any],
|
||
quarter_cad: str,
|
||
snapshot_date: str,
|
||
) -> bool:
|
||
"""UPSERT one building feature → nspd_buildings.
|
||
|
||
Returns True если строка вставлена/обновлена, False при пропуске (нет cad_num).
|
||
"""
|
||
props = feature.get("properties") or {}
|
||
cad_num = props.get("cad_num") or props.get("cadastral_number")
|
||
if not cad_num:
|
||
return False
|
||
|
||
geom = feature.get("geometry")
|
||
geom_json: str | None = json.dumps(geom, ensure_ascii=False) if geom else None
|
||
|
||
try:
|
||
with db.begin_nested():
|
||
db.execute(
|
||
_BUILDING_UPSERT_SQL,
|
||
{
|
||
"cad_num": cad_num,
|
||
"quarter_cad": quarter_cad,
|
||
"purpose": props.get("purpose"),
|
||
"floors": _coerce_int(props.get("floors_above_ground")),
|
||
"floors_underground": _coerce_int(props.get("floors_underground")),
|
||
"year_built": _coerce_int(props.get("year_built")),
|
||
"cost_value": _coerce_numeric(props.get("cost_value")),
|
||
"build_record_area": _coerce_numeric(props.get("build_record_area")),
|
||
"address": props.get("address"),
|
||
"geom_json": geom_json,
|
||
"snapshot_date": snapshot_date,
|
||
},
|
||
)
|
||
return True
|
||
except Exception as e:
|
||
logger.warning("denorm building %s failed: %s", cad_num, e)
|
||
return False
|
||
|
||
|
||
# ── Batch helper ───────────────────────────────────────────────────────────────
|
||
|
||
|
||
def denorm_dump(
|
||
db: Session,
|
||
*,
|
||
quarter_cad: str,
|
||
features: list[dict[str, Any]],
|
||
) -> dict[str, int]:
|
||
"""Denorm all features из одного quarter dump → nspd_parcels / nspd_buildings.
|
||
|
||
Итерирует features_json плоский список. Записи с layer="parcels" идут в
|
||
nspd_parcels, layer="buildings" → nspd_buildings. Остальные layers пропускаются.
|
||
|
||
Каждая строка UPSERT'ится в отдельном SAVEPOINT (begin_nested) — ошибка
|
||
одной строки не откатывает весь batch.
|
||
|
||
Args:
|
||
db: SQLAlchemy Session. Caller отвечает за commit/close после вызова.
|
||
quarter_cad: 3-сегментный кадастровый квартал.
|
||
features: плоский list из features_json JSONB (уже декодированный Python list).
|
||
|
||
Returns:
|
||
dict {"parcels": N, "buildings": M, "errors": K} — количество обработанных строк.
|
||
"""
|
||
snapshot_date = datetime.date.today().isoformat()
|
||
parcels_n = 0
|
||
buildings_n = 0
|
||
errors_n = 0
|
||
|
||
for feat in features:
|
||
layer = feat.get("layer", "")
|
||
try:
|
||
if layer == "parcels":
|
||
if denorm_parcel_feature(
|
||
db, feature=feat, quarter_cad=quarter_cad, snapshot_date=snapshot_date
|
||
):
|
||
parcels_n += 1
|
||
else:
|
||
errors_n += 1
|
||
elif layer == "buildings":
|
||
if denorm_building_feature(
|
||
db, feature=feat, quarter_cad=quarter_cad, snapshot_date=snapshot_date
|
||
):
|
||
buildings_n += 1
|
||
else:
|
||
errors_n += 1
|
||
# Остальные layers (territorial_zones, zouit_*, risk_*, ...) — пропускаем
|
||
except Exception as e:
|
||
logger.warning("denorm feature (layer=%s quarter=%s) failed: %s", layer, quarter_cad, e)
|
||
errors_n += 1
|
||
|
||
db.commit()
|
||
logger.info(
|
||
"denorm_dump quarter=%s parcels=%d buildings=%d errors=%d",
|
||
quarter_cad,
|
||
parcels_n,
|
||
buildings_n,
|
||
errors_n,
|
||
)
|
||
return {"parcels": parcels_n, "buildings": buildings_n, "errors": errors_n}
|