"""DOM.РФ catalog-OBJECT scraper (issue #297 sub-task 22d).

Fills ~25 NULL columns in domrf_kn_objects from public SSR catalog page:
  https://наш.дом.рф/сервисы/каталог-новостроек/объект/{obj_id}

kn-API не возвращает: wall_type, energy_eff, ceiling_height_m, parking_*,
playground_*, finishing_variants_count, etc. — все эти поля есть в
__NEXT_DATA__ JSON блоке на странице каталога (Next.js SSR).

Uses BrowserSession from app.services.scrapers.stealth (Playwright + WAF bypass).
Fetches HTML, extracts __NEXT_DATA__ JSON, maps to DB columns,
UPDATE domrf_kn_objects WHERE obj_id = :id (не перетирает kn-API данные).
"""

from __future__ import annotations

import asyncio
import json
import logging
import re
from datetime import date
from typing import Any

from sqlalchemy import text
from sqlalchemy.orm import Session

from app.services.scrapers.stealth import BASE_URL, BrowserSession, WafBlockedError, jitter_sleep

logger = logging.getLogger(__name__)

# URL шаблон страницы объекта в каталоге DOM.РФ.
# Человекочитаемый вид: https://наш.дом.рф/сервисы/каталог-новостроек/объект/{obj_id}
CATALOG_OBJECT_PATH = "/сервисы/каталог-новостроек/объект/{obj_id}"

# JS snippet — аналог _FETCH_HTML_JS из domrf_catalog.py.
# Выполняется внутри живой Playwright-страницы, возвращает HTML текст.
_FETCH_HTML_JS = """
async ({url}) => {
  try {
    const r = await fetch(url, {credentials: 'include'});
    const ctype = r.headers.get('content-type') || '';
    const body = await r.text();
    return {ok: r.ok, status: r.status, body, contentType: ctype};
  } catch (e) {
    return {ok: false, status: 0, body: String(e), contentType: ''};
  }
}
"""

# UPDATE SQL — обновляет только catalog-derived поля.
# COALESCE гарантирует что NULL-значение не перетирает уже заполненное поле.
UPDATE_OBJECT_CATALOG_SQL = text(
    """
    UPDATE domrf_kn_objects SET
        obj_class                  = COALESCE(:obj_class, obj_class),
        wall_type                  = COALESCE(:wall_type, wall_type),
        energy_eff                 = COALESCE(:energy_eff, energy_eff),
        section_count              = COALESCE(:section_count, section_count),
        parking_total_slots        = COALESCE(:parking_total_slots, parking_total_slots),
        guest_parking_inside_count = COALESCE(
            :guest_parking_inside_count, guest_parking_inside_count
        ),
        guest_parking_outside_count = COALESCE(
            :guest_parking_outside_count, guest_parking_outside_count
        ),
        ceiling_height_m           = COALESCE(:ceiling_height_m, ceiling_height_m),
        finishing_variants_count   = COALESCE(:finishing_variants_count, finishing_variants_count),
        has_free_planning          = COALESCE(:has_free_planning, has_free_planning),
        avg_flat_area_m2           = COALESCE(:avg_flat_area_m2, avg_flat_area_m2),
        elevators_passenger_count  = COALESCE(
            :elevators_passenger_count, elevators_passenger_count
        ),
        elevators_cargo_count      = COALESCE(:elevators_cargo_count, elevators_cargo_count),
        playground_kids_count      = COALESCE(:playground_kids_count, playground_kids_count),
        playground_sport_count     = COALESCE(:playground_sport_count, playground_sport_count),
        has_bike_paths             = COALESCE(:has_bike_paths, has_bike_paths),
        trash_areas_count          = COALESCE(:trash_areas_count, trash_areas_count),
        has_ramp                   = COALESCE(:has_ramp, has_ramp),
        has_low_platforms          = COALESCE(:has_low_platforms, has_low_platforms),
        has_wheelchair_lift        = COALESCE(:has_wheelchair_lift, has_wheelchair_lift),
        first_floor_type           = COALESCE(:first_floor_type, first_floor_type),
        parking_provision_pct      = COALESCE(:parking_provision_pct, parking_provision_pct),
        project_published_at       = COALESCE(:project_published_at, project_published_at),
        project_declaration_num    = COALESCE(:project_declaration_num, project_declaration_num),
        domrf_score_infrastructure = COALESCE(
            :domrf_score_infrastructure, domrf_score_infrastructure
        ),
        domrf_score_transport      = COALESCE(:domrf_score_transport, domrf_score_transport),
        catalog_scraped_at         = NOW()
    WHERE obj_id = :obj_id
      AND snapshot_date = :snapshot_date
    """
)


# ── Value helpers ─────────────────────────────────────────────────────────────


def _to_numeric_comma(s: Any) -> float | None:
    """Конвертировать строку с запятой-десятичным разделителем в float.

    Примеры: "2,7" → 2.7; "2.7" → 2.7; "" → None; None → None.
    """
    if s is None:
        return None
    raw = str(s).strip().replace(",", ".")
    if not raw:
        return None
    try:
        return float(raw)
    except ValueError:
        return None


def _to_date_ddmmyyyy(s: Any) -> date | None:
    """Конвертировать строку "DD.MM.YYYY" в date.

    Примеры: "31.03.2025" → date(2025, 3, 31); "" → None; invalid → None.
    """
    if not s:
        return None
    raw = str(s).strip()
    if not raw:
        return None
    try:
        parts = raw.split(".")
        if len(parts) == 3:
            return date(int(parts[2]), int(parts[1]), int(parts[0]))
    except (ValueError, IndexError):
        pass
    return None


def _to_bool_int(v: Any) -> bool | None:
    """Конвертировать 0/1 (или любое int-like) в bool.

    Примеры: 1 → True; 0 → False; None → None; 3 → True (>0).
    """
    if v is None:
        return None
    try:
        return int(v) > 0
    except (ValueError, TypeError):
        return None


def _to_bool_da_net(s: Any) -> bool | None:
    """Конвертировать "Да"/"Нет" строку в bool.

    Примеры: "Да" → True; "Нет" → False; "" → None; None → None.
    """
    if s is None:
        return None
    raw = str(s).strip().lower()
    if raw == "да":
        return True
    if raw == "нет":
        return False
    return None


def _safe_int(v: Any) -> int | None:
    """Безопасная конвертация в int, None при ошибке."""
    if v is None:
        return None
    try:
        return int(v)
    except (ValueError, TypeError):
        return None


# ── HTML fetching ─────────────────────────────────────────────────────────────


async def fetch_catalog_object_html(session: BrowserSession, obj_id: int) -> str:
    """Получить SSR-HTML страницы объекта в каталоге DOM.РФ.

    Использует тот же паттерн что fetch_catalog_html из domrf_catalog.py:
    fetch() внутри живой Playwright-страницы — WAF-fingerprint идентичен браузеру.

    Raises:
        WafBlockedError: если вернулся не-HTML (JS-challenge или JSON).
        RuntimeError: при 404 или исчерпании попыток.
    """
    if session._page is None:
        raise RuntimeError("BrowserSession not bootstrapped")

    url = BASE_URL + CATALOG_OBJECT_PATH.format(obj_id=obj_id)
    last_err: Exception | None = None

    for attempt in range(5):
        async with session._sem:
            await jitter_sleep(300, 700)
            try:
                session._request_count += 1
                result = await session._page.evaluate(_FETCH_HTML_JS, {"url": url})
            except Exception as exc:
                last_err = exc
                logger.warning(
                    "catalog_object html evaluate err attempt=%d obj_id=%d: %r",
                    attempt,
                    obj_id,
                    exc,
                )
                await asyncio.sleep(2**attempt)
                continue

        status: int = result.get("status", 0)
        body: str = result.get("body", "")
        ctype: str = result.get("contentType", "")

        if status in (429,) or status >= 500 or status == 0:
            last_err = RuntimeError(f"transient status={status}")
            logger.warning(
                "catalog_object transient status=%d attempt=%d obj_id=%d, backing off",
                status,
                attempt,
                obj_id,
            )
            await asyncio.sleep(2**attempt)
            continue

        if status == 404:
            raise RuntimeError(f"catalog_object 404 for obj_id={obj_id}")

        if status != 200:
            raise RuntimeError(f"catalog_object http {status}: {body[:200]} obj_id={obj_id}")

        # Проверяем что вернулся HTML, а не WAF JS-challenge.
        is_html = "text/html" in ctype or "<!doctype" in body[:100].lower()
        if body and not is_html:
            raise WafBlockedError(
                f"non-HTML response for obj_id={obj_id}: status={status} ctype={ctype!r}"
                f" body[:120]={body[:120]!r}"
            )

        if not body:
            raise RuntimeError(f"catalog_object empty body for obj_id={obj_id}")

        return body

    raise RuntimeError(f"catalog_object html max retries exhausted obj_id={obj_id}: {last_err!r}")


# ── __NEXT_DATA__ extraction ──────────────────────────────────────────────────


def extract_next_data(html: str) -> dict[str, Any]:
    """Извлечь JSON из тега <script id="__NEXT_DATA__"> в SSR HTML.

    Raises:
        ValueError: если тег не найден или JSON не парсится.
    """
    match = re.search(
        r'<script\s+id=["\']__NEXT_DATA__["\'][^>]*>(.+?)</script>',
        html,
        re.DOTALL,
    )
    if not match:
        raise ValueError("__NEXT_DATA__ script tag not found in HTML")

    raw_json = match.group(1).strip()
    try:
        return json.loads(raw_json)  # type: ignore[no-any-return]
    except json.JSONDecodeError as exc:
        raise ValueError(f"__NEXT_DATA__ JSON parse error: {exc}") from exc


# ── Field mapping ─────────────────────────────────────────────────────────────


def parse_catalog_object(next_data: dict[str, Any]) -> dict[str, Any]:
    """Извлечь поля объекта из __NEXT_DATA__ и вернуть dict для UPDATE.

    Все .get() безопасны — partial responses OK, отсутствующие поля = None.
    Возвращает dict с bind-параметрами для UPDATE_OBJECT_CATALOG_SQL.
    """
    pp: dict[str, Any] = next_data.get("props", {}).get("pageProps", {})
    ai: dict[str, Any] = pp.get("additionalInfo") or {}
    quart: dict[str, Any] = pp.get("quartography") or {}
    indexes: dict[str, Any] = pp.get("indexes") or {}
    decl: dict[str, Any] = pp.get("projectDeclaration") or {}

    # first_floor_type: 1 = нежилой, 0 = жилой
    first_floor_raw = quart.get("nonLivFirstFloor")
    first_floor_type: str | None = None
    if first_floor_raw is not None:
        try:
            first_floor_type = "нежилой" if int(first_floor_raw) == 1 else "жилой"
        except (ValueError, TypeError):
            pass

    # elevators_cargo_count = cargoElevatorsCount + cargoPassengerElevatorCount
    cargo = _safe_int(ai.get("cargoElevatorsCount"))
    cargo_pass = _safe_int(ai.get("cargoPassengerElevatorCount"))
    if cargo is not None or cargo_pass is not None:
        elevators_cargo_count: int | None = (cargo or 0) + (cargo_pass or 0)
    else:
        elevators_cargo_count = None

    return {
        "obj_class": pp.get("buildingClass"),
        "wall_type": pp.get("wallMaterial"),
        "energy_eff": pp.get("objEnergyEfficiency"),
        "section_count": _safe_int(quart.get("objLivElemEntrCnt")),
        "parking_total_slots": _safe_int(pp.get("parkingCount")),
        "guest_parking_inside_count": _safe_int(ai.get("objectParkingPlaces")),
        "guest_parking_outside_count": _safe_int(ai.get("nearbyParkingPlaces")),
        "ceiling_height_m": _to_numeric_comma(ai.get("ceilingHeight")),
        "finishing_variants_count": _safe_int(pp.get("finishTypeCount")),
        "has_free_planning": _to_bool_da_net(pp.get("freePlan")),
        "avg_flat_area_m2": _to_numeric_comma(quart.get("objLivElemSqAvg")),
        "elevators_passenger_count": _safe_int(ai.get("passengerElevatorsCount")),
        "elevators_cargo_count": elevators_cargo_count,
        "playground_kids_count": _safe_int(ai.get("playgroundsCount")),
        "playground_sport_count": _safe_int(ai.get("sportsgroundCount")),
        "has_bike_paths": _to_bool_int(ai.get("bicycleLane")),
        "trash_areas_count": _safe_int(ai.get("trashAreaCount")),
        "has_ramp": _to_bool_int(ai.get("ramp")),
        "has_low_platforms": _to_bool_int(ai.get("curbLowering")),
        "has_wheelchair_lift": _to_bool_int(ai.get("wheelchairElevatorsCount")),
        "first_floor_type": first_floor_type,
        "parking_provision_pct": _to_numeric_comma(ai.get("parkingAvailabilityPerc")),
        "project_published_at": _to_date_ddmmyyyy(pp.get("publicationDate")),
        "project_declaration_num": decl.get("number"),
        "domrf_score_infrastructure": _safe_int(indexes.get("infrastructure")),
        "domrf_score_transport": _safe_int(indexes.get("transport")),
        # TODO: obj_checks (6 detailed checks) — separate investigation (task #21).
        # pageProps.isChecked (bool), verificationId, verificationFlg available here
        # but detailed per-check breakdown requires separate API investigation.
    }


# ── DB write ──────────────────────────────────────────────────────────────────


async def scrape_catalog_object(
    db: Session,
    session: BrowserSession,
    obj_id: int,
    snapshot_date: date,
) -> bool:
    """Scrape одного объекта: fetch HTML → extract __NEXT_DATA__ → parse → UPDATE.

    Использует SAVEPOINT (begin_nested) для изоляции per-row ошибок.
    Логирует результат через logger.info.

    Returns:
        True если UPDATE затронул строку, False при ошибке или 0 rows.
    """
    logger.info("catalog_object scrape start obj_id=%d snapshot_date=%s", obj_id, snapshot_date)

    try:
        html = await fetch_catalog_object_html(session, obj_id)
    except WafBlockedError as exc:
        logger.warning("catalog_object WAF blocked obj_id=%d: %s", obj_id, exc)
        return False
    except Exception as exc:
        logger.warning("catalog_object fetch failed obj_id=%d: %s", obj_id, exc)
        return False

    try:
        next_data = extract_next_data(html)
    except ValueError as exc:
        logger.warning("catalog_object extract_next_data failed obj_id=%d: %s", obj_id, exc)
        return False

    try:
        data = parse_catalog_object(next_data)
    except Exception as exc:
        logger.warning("catalog_object parse failed obj_id=%d: %s", obj_id, exc)
        return False

    fields_extracted = len([v for v in data.values() if v is not None])

    params: dict[str, Any] = {
        "obj_id": obj_id,
        "snapshot_date": snapshot_date,
        **data,
    }

    try:
        with db.begin_nested():
            result = db.execute(UPDATE_OBJECT_CATALOG_SQL, params)
        rows_affected: int = result.rowcount or 0
    except Exception as exc:
        logger.warning("catalog_object UPDATE failed obj_id=%d: %s", obj_id, exc)
        return False

    if rows_affected == 0:
        logger.warning(
            "catalog_object UPDATE 0 rows obj_id=%d snapshot_date=%s — not in DB?",
            obj_id,
            snapshot_date,
        )
        return False

    logger.info(
        "catalog_object scraped obj_id=%d fields=%d rows_updated=%d",
        obj_id,
        fields_extracted,
        rows_affected,
    )
    return True


async def scrape_catalog_objects(
    db: Session,
    obj_ids: list[int],
    snapshot_date: date,
    region_code: int = 66,
) -> dict[str, int]:
    """Scrape списка объектов через один BrowserSession.

    Запускает один BrowserSession на весь batch; jitter_sleep (300–700 мс)
    встроен в fetch_catalog_object_html для защиты от rate-limit.

    Returns:
        {"processed": N, "succeeded": N, "failed": N, "skipped": N}
    """
    stats: dict[str, int] = {
        "processed": 0,
        "succeeded": 0,
        "failed": 0,
        "skipped": 0,
    }

    if not obj_ids:
        logger.info("scrape_catalog_objects: empty list, nothing to do")
        return stats

    logger.info(
        "scrape_catalog_objects: starting %d objects region=%d snapshot_date=%s",
        len(obj_ids),
        region_code,
        snapshot_date,
    )

    async with BrowserSession(
        region_code=region_code,
        # Страницы каталога публичные — Basic auth не нужен
        auth=None,
    ) as session:
        # Warm-up: visit /сервисы/каталог-новостроек/ to obtain WAF cookies.
        # DOM.РФ WAF (2026-05-24) блокирует SSR-страницы объектов без этих cookies.
        # Idempotent — один вызов покрывает весь batch через этот BrowserSession.
        await session.warm_up()

        for obj_id in obj_ids:
            stats["processed"] += 1
            ok = await scrape_catalog_object(db, session, obj_id, snapshot_date)
            if ok:
                stats["succeeded"] += 1
            else:
                stats["failed"] += 1

    # Commit outer transaction: SAVEPOINT (`begin_nested`) releases внутри loop,
    # но outer tx остаётся autobegin'd — без commit() все UPDATE'ы откатятся
    # при db.close() в Celery task.
    try:
        db.commit()
    except Exception:
        db.rollback()
        raise

    logger.info(
        "scrape_catalog_objects done: processed=%d succeeded=%d failed=%d skipped=%d",
        stats["processed"],
        stats["succeeded"],
        stats["failed"],
        stats["skipped"],
    )
    return stats