"""СберИндекс city-level secondary-housing price index pull (#887 + #794).

Fetches monthly time-series data from sberindex.ru/api/sowa (reverse-engineered
in vault research note 2026-05-31-0845-resolver-794) across three dashboards:
  - real_estate_deals              — hedonic sold-price index
  - residential_real_estate_prices — repeat-sales hedonic (cleanest signal)
  - dinamika-tsen-obyavlenii       — asking-price benchmark

API summary:
  POST https://sberindex.ru/api/sowa
  Body: {"SOWA": {"method": "GET", "route": "<base64>", "data": {"type": "object", "value": []}}}
  route = base64( /dataset/v1/<dashboard>?filter=<urlencode(json)>&limit=1000&offset=0 )
  filter = per-dashboard (see SBER_DASHBOARDS below); common keys: REF_AREA + FREQ:M
  Required headers: content-type, rquid (random 32-hex), x-language, user-agent.

Response cells: __string__<b64> or __number__<raw>. Field names read from 'fields' entry.

#794 fix: build_sber_route was hardcoding SOURCE:SI + REALTY:2 for ALL dashboards.
Live-verified 2026-05-31: residential_real_estate_prices needs REAL_ESTATE_NOVELTY,
dinamika-tsen-obyavlenii needs SOURCE:DK + PRICE_TYPE. Filter is now per-dashboard via
SberDashboard dataclass.

#902 fix: residential_real_estate_prices + dinamika-tsen-obyavlenii returned 404 — the
#794 filter dims were wrong. Re-captured live 2026-06-17 via Playwright route-interception
on the sber dashboards (200 OK):
  - data path  = /dataset/v1/<slug>?filter=<urlencode(json)>&limit=1000&offset=0
                 (NOT /dataset/v1/data/<slug>)
  - residential_real_estate_prices: SOURCE=SI, FREQ=M, REAL_ESTATE_NOVELTY=1, and
    REAL_ESTATE_TYPE ∈ {1=новостройка, 2=вторичка}. The #794 code sent NOVELTY=3 (no such
    code) + TYPE=1 → 404. We pull TYPE=2 (вторичка) — the estimator runs вторичка-only and
    sber_price_index PK is (city, period_month, dashboard) = one series per slug.
  - dinamika-tsen-obyavlenii: SOURCE=DK, FREQ=M, PRICE_TYPE ∈ {1, 2} (both 200). We keep
    PRICE_TYPE=1; the slug+SOURCE=DK is the verified-live combo.

REF_AREA codes (sberindex internal region IDs, not ОКАТО/ISO):
  643 = Россия               — confirmed live, vault research note 2026-05-31-0845.
  66  = Свердловская область — verified 2026-05-31 brute-force against /api/sowa
        (region name from ref_area field of response);
        real_estate_deals: 2017-01 51 046 → 2026-04 128 443 руб/м².
  77  = Москва               — verified 2026-05-31 brute-force against /api/sowa
        (region name from ref_area field of response);
        real_estate_deals: 2017-01 148 471 → 2026-04 309 510 руб/м².
"""

from __future__ import annotations

import asyncio
import base64
import json
import logging
import uuid
from collections.abc import AsyncIterator
from dataclasses import dataclass
from datetime import UTC, date, datetime
from urllib.parse import quote

import httpx
from sqlalchemy import text
from sqlalchemy.orm import Session

logger = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# Config — REF_AREA codes
# ---------------------------------------------------------------------------

# REF_AREA code → city label used as the primary key in sber_price_index.
# Codes verified 2026-05-31 by brute-force scan against /api/sowa
# (region name taken from ref_area field of API response; not ОКАТО/ISO).
SBER_REF_AREAS: dict[str, str] = {
    "643": "Россия",
    "66": "Свердловская область",
    "77": "Москва",
}


# ---------------------------------------------------------------------------
# Per-dashboard config (#794 fix — each dashboard needs different filter params)
# ---------------------------------------------------------------------------


@dataclass(frozen=True)
class SberDashboard:
    slug: str
    extra_filter: dict[str, str]  # beyond REF_AREA + FREQ:"M" — verified live 2026-05-31
    segment_field: str  # response field carrying the segment label


# All three series are the secondary-market (вторичка) ряд;
# one segment per dashboard keeps the (city,period_month,dashboard) PK valid.
SBER_DASHBOARDS: list[SberDashboard] = [
    SberDashboard(
        "real_estate_deals",
        {"SOURCE": "SI", "REALTY": "2"},
        "realty",
    ),
    SberDashboard(
        # #902: вторичка hedonic. REAL_ESTATE_TYPE=2 (вторичка), NOVELTY=1.
        # Old #794 dims (TYPE=1, NOVELTY=3) → 404; re-captured live 2026-06-17.
        "residential_real_estate_prices",
        {"SOURCE": "SI", "REAL_ESTATE_TYPE": "2", "REAL_ESTATE_NOVELTY": "1"},
        "real_estate_type",
    ),
    SberDashboard(
        # #902: asking-price benchmark (ДомКлик). SOURCE=DK, PRICE_TYPE=1 —
        # verified-live 200 combo 2026-06-17.
        "dinamika-tsen-obyavlenii",
        {"SOURCE": "DK", "PRICE_TYPE": "1"},
        "price_type",
    ),
]

SBER_API_URL = "https://sberindex.ru/api/sowa"
SBER_HTTP_TIMEOUT = 30.0
_CHROME_UA = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
    "AppleWebKit/537.36 (KHTML, like Gecko) "
    "Chrome/124.0.0.0 Safari/537.36"
)

# ---------------------------------------------------------------------------
# Route builder
# ---------------------------------------------------------------------------


def build_sber_route(dashboard: str, ref_area: str, extra_filter: dict[str, str]) -> str:
    """Build the base64-encoded route for the /api/sowa payload.

    Encodes /dataset/v1/<dashboard>?filter=<urlencode(json)>&limit=1000&offset=0
    where filter = {"REF_AREA": ref_area, "FREQ": "M", **extra_filter}.

    extra_filter is per-dashboard (see SberDashboard.extra_filter); verified live 2026-05-31.
    All filter values are strings per the live-captured payload (vault research note).
    """
    filter_dict = {"REF_AREA": str(ref_area), "FREQ": "M", **extra_filter}
    # urlencode the JSON, then build the path
    filter_str = quote(json.dumps(filter_dict, separators=(",", ":")), safe="")
    path = f"/dataset/v1/{dashboard}?filter={filter_str}&limit=1000&offset=0"
    return base64.b64encode(path.encode("utf-8")).decode("ascii")


# ---------------------------------------------------------------------------
# Response decoder
# ---------------------------------------------------------------------------


def _decode_cell(cell: str) -> str | float:
    """Decode a single response cell.

    Cells arrive as one of:
      '__string__<base64>'  → decode UTF-8 string
      '__number__<raw>'     → parse as float
    """
    if cell.startswith("__string__"):
        encoded = cell[len("__string__") :]
        return base64.b64decode(encoded).decode("utf-8")
    if cell.startswith("__number__"):
        return float(cell[len("__number__") :])
    # Fallback: try numeric, else return raw
    try:
        return float(cell)
    except (ValueError, TypeError):
        return str(cell)


def decode_sber_response(
    response_data: dict,  # type: ignore[type-arg]
) -> list[dict[str, str | float]]:
    """Decode the SOWA response payload into a list of row dicts.

    Raises ValueError if the response structure is unexpected.
    """
    try:
        sowa = response_data["SOWA"]
        values = sowa["data"]["value"]
    except (KeyError, TypeError) as exc:
        raise ValueError(f"Unexpected SOWA response structure: {exc}") from exc

    fields: list[str] | None = None
    rows_raw: list[list[str]] = []

    for entry in values:
        key = entry.get("key")
        val = entry.get("value")
        if key == "fields":
            fields = [_decode_cell(c) for c in val]  # type: ignore[arg-type]
        elif key == "data":
            rows_raw = val

    if fields is None:
        raise ValueError("No 'fields' key found in SOWA response")

    result: list[dict[str, str | float]] = []
    for raw_row in rows_raw:
        if len(raw_row) != len(fields):
            logger.warning(
                "sber_index: row length mismatch (expected %d, got %d) — skip",
                len(fields),
                len(raw_row),
            )
            continue
        row_dict: dict[str, str | float] = {}
        for col_name, cell in zip(fields, raw_row, strict=True):
            row_dict[col_name] = _decode_cell(cell)  # type: ignore[arg-type]
        result.append(row_dict)

    return result


# ---------------------------------------------------------------------------
# Period normalisation
# ---------------------------------------------------------------------------


def _parse_period_month(period_str: str) -> date:
    """Parse the API period string into the first day of the month.

    API returns ISO-8601 with UTC offset, e.g. '2017-01-30T21:00:00Z'.
    We normalise to date(year, month, 1) — the first day of the reported month.
    The day/time portion is an artefact of timezone shift and is discarded.
    """
    dt = datetime.fromisoformat(period_str.replace("Z", "+00:00"))
    # Shift to UTC date, then take first day of that month
    dt_utc = dt.astimezone(UTC)
    return date(dt_utc.year, dt_utc.month, 1)


# ---------------------------------------------------------------------------
# Async fetch — single dashboard × city
# ---------------------------------------------------------------------------


async def fetch_sber_index(
    client: httpx.AsyncClient,
    *,
    dashboard: SberDashboard,
    ref_area: str,
) -> AsyncIterator[tuple[str, date, str, str, float]]:
    """Fetch one dashboard × REF_AREA combination from sberindex.ru/api/sowa.

    Yields tuples of:
        (city_label, period_month, segment_label, dashboard_slug, index_value_rub_m2)

    city_label comes from the API response field 'ref_area'.
    segment_label comes from dashboard.segment_field (per-dashboard, verified 2026-05-31).

    Raises httpx.HTTPStatusError on non-2xx response (caller handles per-series).
    """
    route = build_sber_route(dashboard.slug, ref_area, dashboard.extra_filter)
    rquid = uuid.uuid4().hex  # 32-hex random, required header

    payload = {
        "SOWA": {
            "method": "GET",
            "route": route,
            "data": {"type": "object", "value": []},
        }
    }
    headers = {
        "content-type": "application/json",
        "rquid": rquid,
        "x-language": "ru",
        "accept": "application/json, text/plain, */*",
        "user-agent": _CHROME_UA,
    }

    resp = await client.post(SBER_API_URL, json=payload, headers=headers)
    resp.raise_for_status()

    rows = decode_sber_response(resp.json())

    for row in rows:
        period_raw = row.get("period", "")
        value_raw = row.get("value")
        city_label = str(row.get("ref_area", ref_area))
        segment_label = str(row.get(dashboard.segment_field, ""))

        if not period_raw or value_raw is None:
            continue

        try:
            period_month = _parse_period_month(str(period_raw))
            index_value = float(value_raw)
        except (ValueError, TypeError) as exc:
            logger.warning(
                "sber_index: skip row — parse error for dashboard=%s ref_area=%s: %s",
                dashboard.slug,
                ref_area,
                exc,
            )
            continue

        yield (city_label, period_month, segment_label, dashboard.slug, index_value)


# ---------------------------------------------------------------------------
# Bulk pull + upsert
# ---------------------------------------------------------------------------


def _upsert_rows_sync(db: Session, rows_to_upsert: list[tuple[str, date, str, str, float]]) -> None:
    """Synchronous per-row UPSERT into sber_price_index + commit.

    #1348: blocking psycopg work — must run via asyncio.to_thread, never directly
    on the event loop. Idempotent ON CONFLICT(city, period_month, dashboard).
    """
    for city_label, period_month, segment, dash, value in rows_to_upsert:
        db.execute(
            text(
                """
                INSERT INTO sber_price_index
                    (city, period_month, segment, dashboard,
                     index_value_rub_m2, source, fetched_at)
                VALUES (
                    CAST(:city AS text),
                    CAST(:period_month AS date),
                    CAST(:segment AS text),
                    CAST(:dashboard AS text),
                    CAST(:index_value AS double precision),
                    'sberindex',
                    now()
                )
                ON CONFLICT (city, period_month, dashboard)
                DO UPDATE SET
                    index_value_rub_m2 = EXCLUDED.index_value_rub_m2,
                    segment            = EXCLUDED.segment,
                    fetched_at         = now()
                """
            ),
            {
                "city": city_label,
                "period_month": period_month.isoformat(),
                "segment": segment,
                "dashboard": dash,
                "index_value": value,
            },
        )

    db.commit()


def _query_our_median_sync(db: Session) -> float | None:
    """Synchronous benchmark SELECT — scrape-median asking ppm² (ЕКБ вторичка).

    #1348: blocking psycopg work — must run via asyncio.to_thread.
    """
    return db.execute(
        text(
            # novostroyki guard (#1186): NULL = legacy вторичка до м.011
            "SELECT percentile_cont(0.5)"
            " WITHIN GROUP (ORDER BY price_per_m2) AS m"
            " FROM listings"
            " WHERE is_active = true"
            " AND price_per_m2 BETWEEN 50000 AND 800000"
            " AND (listing_segment IS NULL"
            "      OR listing_segment = 'vtorichka')"
        )
    ).scalar()


async def pull_sber_indices(
    db: Session,
    *,
    cities: dict[str, str] | None = None,
    dashboards: list[SberDashboard] | None = None,
) -> dict[str, int]:
    """Fetch all configured dashboards × cities and upsert into sber_price_index.

    Args:
        db: SQLAlchemy Session (tradein DB).
        cities: REF_AREA code → city label mapping. Defaults to SBER_REF_AREAS.
        dashboards: list of SberDashboard configs to pull. Defaults to SBER_DASHBOARDS.

    Returns dict of counters: {upserted: int, skipped: int, errors: int}.

    Per-series try/except: one failing series logs + continues (does not abort others).
    After upserting 'dinamika-tsen-obyavlenii', logs the latest asking index per city
    as a data-quality benchmark reference and compares to our scrape-median asking ppm².
    """
    if cities is None:
        cities = SBER_REF_AREAS
    if dashboards is None:
        dashboards = SBER_DASHBOARDS

    counters: dict[str, int] = {"upserted": 0, "skipped": 0, "errors": 0}

    # #922: sberindex.ru serves an INCOMPLETE TLS chain — it sends only the leaf
    # certificate and omits the Let's Encrypt intermediate (R13), so cert
    # verification fails with "unable to get local issuer certificate" (verify
    # code 21) even against a full system/certifi trust store. This is a
    # server-side misconfiguration outside our control. The endpoint is public,
    # unauthenticated, non-sensitive open data, so we skip TLS verification here
    # rather than pin a Let's Encrypt intermediate that rotates (R10–R14+).
    async with httpx.AsyncClient(timeout=SBER_HTTP_TIMEOUT, verify=False) as client:
        for ref_area, _city_hint in cities.items():
            for dashboard in dashboards:
                try:
                    rows_to_upsert: list[tuple[str, date, str, str, float]] = []
                    async for row in fetch_sber_index(
                        client, dashboard=dashboard, ref_area=ref_area
                    ):
                        rows_to_upsert.append(row)

                    if not rows_to_upsert:
                        logger.info(
                            "sber_index: no rows returned for dashboard=%s ref_area=%s",
                            dashboard.slug,
                            ref_area,
                        )
                        counters["skipped"] += 1
                        continue

                    # Idempotent upsert — ON CONFLICT(city, period_month, dashboard).
                    # #1348: the blocking psycopg UPSERT loop + commit run off the
                    # event loop via asyncio.to_thread (sync Session is fine here —
                    # access is sequential, never concurrent).
                    await asyncio.to_thread(_upsert_rows_sync, db, rows_to_upsert)
                    counters["upserted"] += len(rows_to_upsert)

                    logger.info(
                        "sber_index: upserted %d rows for dashboard=%s ref_area=%s",
                        len(rows_to_upsert),
                        dashboard.slug,
                        ref_area,
                    )

                    # Data-quality benchmark log for asking-price series
                    if dashboard.slug == "dinamika-tsen-obyavlenii" and rows_to_upsert:
                        latest_row = max(rows_to_upsert, key=lambda r: r[1])
                        city_lbl, latest_month, _, _, latest_val = latest_row
                        logger.info(
                            "sber_index benchmark [asking]: city=%r latest_month=%s "
                            "index_value_rub_m2=%.0f "
                            "(data-quality ref; full asking-vs-our-median below)",
                            city_lbl,
                            latest_month.isoformat(),
                            latest_val,
                        )
                        # our listings median is ЕКБ-only →
                        # compare only against Свердловская обл. (REF_AREA 66)
                        if ref_area == "66":
                            # Task B: best-effort reconciliation to our scrape-median asking ppm²
                            try:
                                # #1348: blocking benchmark SELECT off the event loop
                                our_median = await asyncio.to_thread(_query_our_median_sync, db)
                                if our_median is not None and latest_val:
                                    sber_latest = latest_val
                                    divergence_pct = (our_median - sber_latest) / sber_latest * 100
                                    logger.info(
                                        "sber_index benchmark [asking vs our median]: "
                                        "sber=%.0f our_median=%.0f divergence=%+.1f%% "
                                        "(data-quality signal)",
                                        sber_latest,
                                        our_median,
                                        divergence_pct,
                                    )
                            except Exception as exc:
                                logger.debug(
                                    "sber benchmark reconciliation skipped (graceful): %s", exc
                                )

                except httpx.HTTPStatusError as exc:
                    # #902: a 404 on /api/sowa almost always means the /dataset/v1/<slug>
                    # path or its filter dims are stale (sber renames slugs / changes
                    # dimension codes). Surface it loudly with the slug + filter so the
                    # next breakage is diagnosable instead of a silent error-counter bump.
                    if exc.response.status_code == 404:
                        logger.warning(
                            "sber_index: 404 for dashboard=%s ref_area=%s filter=%s — "
                            "dataset-path invalid? slug renamed or filter dims stale "
                            "(re-capture /dataset/v1/<slug> via dashboard route-interception)",
                            dashboard.slug,
                            ref_area,
                            {"REF_AREA": ref_area, "FREQ": "M", **dashboard.extra_filter},
                        )
                    else:
                        logger.error(
                            "sber_index: HTTP %d for dashboard=%s ref_area=%s — skip series",
                            exc.response.status_code,
                            dashboard.slug,
                            ref_area,
                        )
                    counters["errors"] += 1
                except httpx.RequestError as exc:
                    logger.error(
                        "sber_index: network error for dashboard=%s ref_area=%s: %s — skip",
                        dashboard.slug,
                        ref_area,
                        exc,
                    )
                    counters["errors"] += 1
                except Exception:
                    # #1345: a failed db.execute leaves the shared Session in an
                    # aborted-transaction state; without rollback every remaining
                    # series cascades into InFailedSqlTransaction. Roll back so the
                    # next (city, dashboard) iteration starts on a clean transaction.
                    # #1348: rollback is blocking psycopg work — keep it off the loop.
                    await asyncio.to_thread(db.rollback)
                    logger.exception(
                        "sber_index: unexpected error for dashboard=%s ref_area=%s — skip",
                        dashboard.slug,
                        ref_area,
                    )
                    counters["errors"] += 1

    return counters
