gendesign/backend/app/services/scrapers/nspd_denorm.py

"""Denormalize nspd_quarter_dumps.features_json → nspd_parcels / nspd_buildings.

Используется:
  - Inline в harvest_quarter task после UPSERT dump'а
  - Backfill task для existing dumps (backfill_all_dumps)

Property mapping (NSPD WMS GetFeatureInfo responses):
  Parcels (layer="parcels", layer id 36048):
    cad_num          — кадастровый номер ЗУ
    permitted_use    — ВРИ
    land_category    — категория земель
    cost_value       — кадастровая стоимость, руб.
    area             — площадь, м²
    address          — адрес

  Buildings (layer="buildings", layer id 36049):
    cad_num                — кадастровый номер ОКС
    purpose                — назначение ("Многоквартирный дом" / "Нежилое здание" / ...)
    floors_above_ground    — надземных этажей
    floors_underground     — подземных этажей
    year_built             — год постройки
    cost_value             — кадастровая стоимость, руб.
    build_record_area      — площадь по ГКН, м²
    address                — адрес

Геометрия в features_json хранится в EPSG:3857 (как возвращает NSPD WMS) —
трансформация 3857→4326 делается в SQL через ST_Transform.
"""

from __future__ import annotations

import datetime
import json
import logging
import re
from typing import Any

from sqlalchemy import text
from sqlalchemy.orm import Session

logger = logging.getLogger(__name__)

# ── Engineering classifier ─────────────────────────────────────────────────────

# Паттерны для классификации инженерных сооружений (layer 36328) по текстовым
# свойствам. Источник: bug-post-mortem fixes/Bug_NSPD_WMS_NotBulk_2026_May14.md
# + live test данные 7×7 grid на 1км² центра ЕКБ.
# Порядок проверки важен: более специфичные паттерны идут первыми.

_ENGINEERING_PATTERNS: list[tuple[re.Pattern[str], str]] = [
    # gas — газопроводы, ГРП, ГК (газовые колодцы)
    (re.compile(r"газопровод|газоснабж|ГРП\b|ГК\b", re.IGNORECASE), "gas"),
    # sewage — канализация, сток, ливневые сети
    (re.compile(r"канализ|сточ|ливнев|самотёч|самотеч", re.IGNORECASE), "sewage"),
    # heat — тепловые сети, котельные, ТЭЦ
    (
        re.compile(r"теплов(ая|ой|ые)|теплосеть|теплоснабж|котельн|ТЭЦ\b", re.IGNORECASE),
        "heat",
    ),
    # electric — электросети, ВЛ, КЛ, ЛЭП, подстанции, трансформаторы, ТП
    (
        re.compile(
            r"электроэнерг|ВЛ[\s\-]|ВЛ-?\d|КЛ[\s\-]|КЛ-?\d|ЛЭП\b"
            r"|подстанц|трансформ|ТП[\s\-]?\d",
            re.IGNORECASE,
        ),
        "electric",
    ),
    # water — водопровод, водоснабжение, хозбытовые водосети
    (re.compile(r"водопровод|водоснабж|хозбытов|водовод", re.IGNORECASE), "water"),
]

# Поля из NSPD properties в которых ищем паттерны (по приоритету)
_ENGINEERING_TEXT_FIELDS = ("params_name", "name", "params_purpose", "purpose", "label")


def classify_engineering_kind(properties: dict[str, Any]) -> str:
    """Классифицировать инженерное сооружение (layer 36328) по его properties.

    Проверяет поля `params_name`, `name`, `params_purpose`, `purpose`, `label`
    против regex-паттернов. Возвращает первое совпадение.

    Args:
        properties: dict свойств NSPDFeature.properties из WMS GetFeatureInfo
                    или search/geoportal response.

    Returns:
        Одно из: ``"water"`` | ``"sewage"`` | ``"gas"`` | ``"heat"``
        | ``"electric"`` | ``"other"``.

    Examples:
        >>> classify_engineering_kind({"params_name": "Газопровод высокого давления"})
        'gas'
        >>> classify_engineering_kind({"name": "КЛ 10 кВ ТП 64102"})
        'electric'
        >>> classify_engineering_kind({"params_purpose": "Водопровод хозбытовой"})
        'water'
        >>> classify_engineering_kind({"params_name": "Тепловая сеть"})
        'heat'
        >>> classify_engineering_kind({"name": "Канализация"})
        'sewage'
    """
    # Собираем текст для проверки из всех релевантных полей
    text_parts: list[str] = []
    for field in _ENGINEERING_TEXT_FIELDS:
        val = properties.get(field)
        if val and isinstance(val, str):
            text_parts.append(val)

    combined = " ".join(text_parts)
    if not combined:
        return "other"

    for pattern, kind in _ENGINEERING_PATTERNS:
        if pattern.search(combined):
            return kind

    return "other"


# ── Type coercions ─────────────────────────────────────────────────────────────


def _coerce_int(v: Any) -> int | None:
    """NSPD properties могут быть str / int / None."""
    if v is None:
        return None
    try:
        return int(v)
    except (ValueError, TypeError):
        return None


def _coerce_numeric(v: Any) -> float | None:
    """NSPD properties могут быть str с запятой (европейский формат) или None."""
    if v is None:
        return None
    try:
        return float(str(v).replace(",", ".").strip())
    except (ValueError, TypeError):
        return None


# ── Parcel UPSERT ──────────────────────────────────────────────────────────────

_PARCEL_UPSERT_SQL = text(
    """
    INSERT INTO nspd_parcels (
        cad_num, quarter_cad, permitted_use, land_category,
        cost_value, cost_per_m2, area_sqm, address, geom, snapshot_date
    ) VALUES (
        CAST(:cad_num AS text),
        CAST(:quarter_cad AS text),
        CAST(:permitted_use AS text),
        CAST(:land_category AS text),
        CAST(:cost_value AS numeric),
        CAST(:cost_per_m2 AS numeric),
        CAST(:area_sqm AS numeric),
        CAST(:address AS text),
        CASE WHEN :geom_json IS NULL THEN NULL
             ELSE ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(:geom_json), 3857), 4326)
        END,
        CAST(:snapshot_date AS date)
    )
    ON CONFLICT (cad_num) DO UPDATE SET
        quarter_cad   = EXCLUDED.quarter_cad,
        permitted_use = EXCLUDED.permitted_use,
        land_category = EXCLUDED.land_category,
        cost_value    = EXCLUDED.cost_value,
        cost_per_m2   = EXCLUDED.cost_per_m2,
        area_sqm      = EXCLUDED.area_sqm,
        address       = EXCLUDED.address,
        geom          = EXCLUDED.geom,
        snapshot_date = EXCLUDED.snapshot_date,
        updated_at    = NOW()
    """
)


def denorm_parcel_feature(
    db: Session,
    *,
    feature: dict[str, Any],
    quarter_cad: str,
    snapshot_date: str,
) -> bool:
    """UPSERT one parcel feature → nspd_parcels.

    Returns True если строка вставлена/обновлена, False при пропуске (нет cad_num).
    """
    props = feature.get("properties") or {}
    cad_num = props.get("cad_num") or props.get("cadastral_number")
    if not cad_num:
        return False

    area = _coerce_numeric(props.get("area"))
    cost_value = _coerce_numeric(props.get("cost_value"))
    cost_per_m2: float | None = None
    if cost_value is not None and area is not None and area > 0:
        cost_per_m2 = cost_value / area

    geom = feature.get("geometry")
    geom_json: str | None = json.dumps(geom, ensure_ascii=False) if geom else None

    try:
        with db.begin_nested():
            db.execute(
                _PARCEL_UPSERT_SQL,
                {
                    "cad_num": cad_num,
                    "quarter_cad": quarter_cad,
                    "permitted_use": props.get("permitted_use"),
                    "land_category": props.get("land_category"),
                    "cost_value": cost_value,
                    "cost_per_m2": cost_per_m2,
                    "area_sqm": area,
                    "address": props.get("address"),
                    "geom_json": geom_json,
                    "snapshot_date": snapshot_date,
                },
            )
        return True
    except Exception as e:
        logger.warning("denorm parcel %s failed: %s", cad_num, e)
        return False


# ── Building UPSERT ────────────────────────────────────────────────────────────

_BUILDING_UPSERT_SQL = text(
    """
    INSERT INTO nspd_buildings (
        cad_num, quarter_cad, purpose, floors, floors_underground,
        year_built, cost_value, build_record_area, address, geom, snapshot_date
    ) VALUES (
        CAST(:cad_num AS text),
        CAST(:quarter_cad AS text),
        CAST(:purpose AS text),
        CAST(:floors AS integer),
        CAST(:floors_underground AS integer),
        CAST(:year_built AS integer),
        CAST(:cost_value AS numeric),
        CAST(:build_record_area AS numeric),
        CAST(:address AS text),
        CASE WHEN :geom_json IS NULL THEN NULL
             ELSE ST_Transform(ST_SetSRID(ST_GeomFromGeoJSON(:geom_json), 3857), 4326)
        END,
        CAST(:snapshot_date AS date)
    )
    ON CONFLICT (cad_num) DO UPDATE SET
        quarter_cad       = EXCLUDED.quarter_cad,
        purpose           = EXCLUDED.purpose,
        floors            = EXCLUDED.floors,
        floors_underground = EXCLUDED.floors_underground,
        year_built        = EXCLUDED.year_built,
        cost_value        = EXCLUDED.cost_value,
        build_record_area = EXCLUDED.build_record_area,
        address           = EXCLUDED.address,
        geom              = EXCLUDED.geom,
        snapshot_date     = EXCLUDED.snapshot_date,
        updated_at        = NOW()
    """
)


def denorm_building_feature(
    db: Session,
    *,
    feature: dict[str, Any],
    quarter_cad: str,
    snapshot_date: str,
) -> bool:
    """UPSERT one building feature → nspd_buildings.

    Returns True если строка вставлена/обновлена, False при пропуске (нет cad_num).
    """
    props = feature.get("properties") or {}
    cad_num = props.get("cad_num") or props.get("cadastral_number")
    if not cad_num:
        return False

    geom = feature.get("geometry")
    geom_json: str | None = json.dumps(geom, ensure_ascii=False) if geom else None

    try:
        with db.begin_nested():
            db.execute(
                _BUILDING_UPSERT_SQL,
                {
                    "cad_num": cad_num,
                    "quarter_cad": quarter_cad,
                    "purpose": props.get("purpose"),
                    "floors": _coerce_int(props.get("floors_above_ground")),
                    "floors_underground": _coerce_int(props.get("floors_underground")),
                    "year_built": _coerce_int(props.get("year_built")),
                    "cost_value": _coerce_numeric(props.get("cost_value")),
                    "build_record_area": _coerce_numeric(props.get("build_record_area")),
                    "address": props.get("address"),
                    "geom_json": geom_json,
                    "snapshot_date": snapshot_date,
                },
            )
        return True
    except Exception as e:
        logger.warning("denorm building %s failed: %s", cad_num, e)
        return False


# ── Batch helper ───────────────────────────────────────────────────────────────


def denorm_dump(
    db: Session,
    *,
    quarter_cad: str,
    features: list[dict[str, Any]],
) -> dict[str, int]:
    """Denorm all features из одного quarter dump → nspd_parcels / nspd_buildings.

    Итерирует features_json плоский список. Записи с layer="parcels" идут в
    nspd_parcels, layer="buildings" → nspd_buildings. Остальные layers пропускаются.

    Каждая строка UPSERT'ится в отдельном SAVEPOINT (begin_nested) — ошибка
    одной строки не откатывает весь batch.

    Args:
        db: SQLAlchemy Session. Caller отвечает за commit/close после вызова.
        quarter_cad: 3-сегментный кадастровый квартал.
        features: плоский list из features_json JSONB (уже декодированный Python list).

    Returns:
        dict {"parcels": N, "buildings": M, "errors": K, "skipped": S} —
        количество обработанных строк. ``errors`` — только реальные сбои UPSERT;
        ``skipped`` — штатные пропуски feature без cad_num.
    """
    snapshot_date = datetime.date.today().isoformat()
    parcels_n = 0
    buildings_n = 0
    errors_n = 0
    skipped_n = 0

    for feat in features:
        layer = feat.get("layer", "")
        try:
            if layer == "parcels":
                # denorm_parcel_feature возвращает False и при штатном пропуске
                # (нет cad_num), и при реальном сбое UPSERT — различаем их здесь,
                # чтобы не завышать error-счётчик прогона.
                props = feat.get("properties") or {}
                if not (props.get("cad_num") or props.get("cadastral_number")):
                    skipped_n += 1
                elif denorm_parcel_feature(
                    db, feature=feat, quarter_cad=quarter_cad, snapshot_date=snapshot_date
                ):
                    parcels_n += 1
                else:
                    errors_n += 1
            elif layer == "buildings":
                props = feat.get("properties") or {}
                if not (props.get("cad_num") or props.get("cadastral_number")):
                    skipped_n += 1
                elif denorm_building_feature(
                    db, feature=feat, quarter_cad=quarter_cad, snapshot_date=snapshot_date
                ):
                    buildings_n += 1
                else:
                    errors_n += 1
            # Остальные layers (territorial_zones, zouit_*, risk_*, ...) — пропускаем
        except Exception as e:
            logger.warning("denorm feature (layer=%s quarter=%s) failed: %s", layer, quarter_cad, e)
            errors_n += 1

    db.commit()
    logger.info(
        "denorm_dump quarter=%s parcels=%d buildings=%d errors=%d skipped=%d",
        quarter_cad,
        parcels_n,
        buildings_n,
        errors_n,
        skipped_n,
    )
    return {
        "parcels": parcels_n,
        "buildings": buildings_n,
        "errors": errors_n,
        "skipped": skipped_n,
    }