"""Yandex.Недвижимость scraper (realty.yandex.ru) — DOM-based SERP parser.

History: Yandex SSR used to embed full state in `<script id="initial_state_script">`.
As of 2026-05, the state is chunked + evaporates after hydration. This version
scrapes the rendered DOM via selectolax + helpers from yandex_helpers.py.

Area extraction: DOM concat-text is empty pre-hydration (~82% NULL). Fix: extract
area from the page-level embedded JSON state (script[id="initial_state_script"] or
application/json script), keyed by offerId. DOM regex is kept as fallback.

Transport (T4): curl_cffi AsyncSession(impersonate="chrome120") — Chrome TLS
fingerprint bypasses Yandex anti-bot gate that returns captcha/shell HTML for plain
httpx. Optional cookies loaded from YANDEX_COOKIES_FILE env path.

Combos (T5): fetch_around_multi_room supports rooms × price-range combinations to
work around Yandex SERP ~575-card cap per query. Full combos → ~4000 unique listings.

URL pattern (path-based, no geo radius):
    https://realty.yandex.ru/{city}/kupit/kvartira/[{room-slug}/]vtorichniy-rynok/?...
"""

from __future__ import annotations

import json
import logging
import re
from datetime import date  # noqa: F401  (used in type hints / future helpers)
from pathlib import Path
from typing import Any
from urllib.parse import urlencode

from curl_cffi.requests import AsyncSession as _CurlCffiSession
from selectolax.parser import HTMLParser, Node

from app.services.scraper_settings import get_scraper_delay
from app.services.scrapers.base import BaseScraper, ScrapedLot
from app.services.scrapers.repair_state_normalizer import infer_repair_state_from_text
from app.services.scrapers.yandex_helpers import (
    RE_FLOOR,
    RE_JK_ID,
    RE_OFFER_ID,
    RE_PPM2,
    RE_PRICE,
    RE_TITLE_AREA,
    RE_TITLE_ROOMS,
    parse_listing_date,
    parse_rub,
)

logger = logging.getLogger(__name__)

DEFAULT_CITY = "ekaterinburg"
MAX_PAGES = 3
PHOTO_DOMAIN = "avatars.mds.yandex"
PHOTO_SIZE_FROM = "app_snippet_small"
PHOTO_SIZE_TO = "main"

# Address container wraps street link + house number text node.
# Class is hashed but always starts with "AddressWithGeoLinks__addressContainer".
ADDRESS_CONTAINER_SELECTOR = '[class*="AddressWithGeoLinks__addressContainer"]'
_RE_WS = re.compile(r"\s+")

# Yandex SERP embeds offer data in a <script id="initial_state_script"> as JSON,
# or (since 2026-05) in <script type="application/json" ...> chunks.
# We look for these selectors in priority order.
_STATE_SCRIPT_SELECTORS = [
    'script[id="initial_state_script"]',
    'script[type="application/json"]',
]

# Area field candidates in a Yandex offer object (tried in order).
_AREA_FIELDS = ("totalArea", "area", "spaceTotal", "spaceAll", "squareTotal")

# Captcha detection markers (checked in first 5000 chars, lowercased).
_CAPTCHA_MARKERS = ("/showcaptcha", "smartcaptcha", "checkboxcaptcha")

# Path segments for room buckets (Yandex URL canonical slugs).
# Key matches the rooms label used in combos; value = URL path segment.
ROOM_PATH: dict[str, str] = {
    "studio": "studiya",
    "1": "odnokomnatnaya",
    "2": "dvuhkomnatnaya",
    "3": "trehkomnatnaya",
    "4+": "4-i-bolee",
}

# Default price ranges (price_min, price_max) in rubles. None = open-ended.
DEFAULT_PRICE_RANGES: list[tuple[int | None, int | None]] = [
    (None, 5_000_000),
    (5_000_000, 7_000_000),
    (7_000_000, 10_000_000),
    (10_000_000, 15_000_000),
    (15_000_000, 25_000_000),
    (25_000_000, None),
]


def _load_cookies_from_file(path: str | None) -> dict[str, str]:
    """Load Yandex cookies from JSON file path (Netscape/browser export format).

    Format: [{name: str, value: str, ...}, ...].
    Returns empty dict if path is None, missing, or unparseable — not a failure.
    """
    if not path:
        return {}
    p = Path(path)
    if not p.exists():
        logger.warning("yandex cookies file not found: %s — running without cookies", path)
        return {}
    try:
        raw = json.loads(p.read_text(encoding="utf-8"))
        result = {
            item["name"]: item["value"]
            for item in raw
            if isinstance(item, dict) and "name" in item and "value" in item
        }
        logger.info("yandex: loaded %d cookies from %s", len(result), path)
        return result
    except Exception:
        logger.warning("yandex: failed to parse cookies file %s — running without", path)
        return {}


def _is_captcha(html: str) -> bool:
    """Return True if the response looks like a Yandex captcha page."""
    head = html[:5000].lower()
    return any(m in head for m in _CAPTCHA_MARKERS) or "докажите, что вы не робот" in head


def _extract_offer_areas_from_state(html: str) -> dict[str, float]:
    """Extract {offer_id: area_m2} map from Yandex SERP embedded JSON state.

    Tries each known script selector, parses JSON, then traverses known
    offer-array paths looking for offer objects with an id + area field.

    Returns empty dict if state is absent or unparseable.
    """
    tree = HTMLParser(html)
    candidates: list[str] = []
    for selector in _STATE_SCRIPT_SELECTORS:
        for node in tree.css(selector):
            text = node.text() or ""
            text = text.strip()
            if text:
                candidates.append(text)

    result: dict[str, float] = {}
    for text in candidates:
        try:
            state = json.loads(text)
        except (json.JSONDecodeError, ValueError):
            continue
        if not isinstance(state, dict):
            continue
        _collect_areas_from_state(state, result)
        if result:
            break  # успешно извлекли из первого валидного blob'а
    return result


def _collect_areas_from_state(state: dict[str, Any], out: dict[str, float]) -> None:
    """Walk known Yandex state paths to find offer objects and extract area.

    Known paths (tried in order):
      - state["offers"]                           (flat list)
      - state["pageData"]["offers"]               (SERP v2)
      - state["listing"]["offers"]                (SERP v3)
      - state["serpData"]["items"]                (SERP v4 chunks)
    """
    offer_lists: list[Any] = []
    for path in (
        ["offers"],
        ["pageData", "offers"],
        ["listing", "offers"],
        ["serpData", "items"],
    ):
        node: Any = state
        for key in path:
            if not isinstance(node, dict):
                node = None
                break
            node = node.get(key)
        if isinstance(node, list):
            offer_lists.append(node)

    for offer_list in offer_lists:
        for offer in offer_list:
            if not isinstance(offer, dict):
                continue
            # Flatten one level of nesting (some Yandex shapes wrap the offer)
            inner = offer.get("offer") or offer
            if not isinstance(inner, dict):
                continue
            oid = inner.get("offerId") or inner.get("id")
            if oid is None:
                continue
            area = _read_area_field(inner)
            if area is not None:
                out[str(oid)] = area


def _read_area_field(offer: dict[str, Any]) -> float | None:
    """Try known field names to read area (м²) from a Yandex offer dict."""
    for field in _AREA_FIELDS:
        val = offer.get(field)
        if val is None:
            continue
        # Some fields are nested dicts: {"value": 52.0, "unit": "SQM"}
        if isinstance(val, dict):
            val = val.get("value")
        if val is not None:
            try:
                result = float(val)
                if result > 0:
                    return result
            except (TypeError, ValueError):
                continue
    return None


class YandexRealtyScraper(BaseScraper):
    name = "yandex"
    base_url = "https://realty.yandex.ru"
    request_delay_sec = 5.0

    def __init__(self, city: str = DEFAULT_CITY) -> None:
        super().__init__()
        self.city = city
        # Load global Yandex delay from DB at instantiation
        self.request_delay_sec = get_scraper_delay(self.name)
        self._cffi_session: _CurlCffiSession | None = None
        self._cookies: dict[str, str] = {}

    async def __aenter__(self) -> YandexRealtyScraper:  # type: ignore[override]
        """Override: Chrome120 TLS impersonation bypasses Yandex anti-bot gate.

        Plain httpx returns captcha/shell-HTML on datacenter IPs.
        Sibling pattern: yandex_valuation.py.
        Cookies loaded from YANDEX_COOKIES_FILE settings path (optional).
        """
        # Deferred import to avoid circular / settings-at-import-time issues
        from app.core.config import settings

        self._cookies = _load_cookies_from_file(settings.yandex_cookies_file)
        self._cffi_session = _CurlCffiSession(
            impersonate="chrome120",
            cookies=self._cookies or {},
            headers={
                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
                "Accept-Encoding": "gzip, deflate, br",
                "Connection": "keep-alive",
                "Upgrade-Insecure-Requests": "1",
            },
        )
        return self

    async def __aexit__(self, *args: object) -> None:  # type: ignore[override]
        if self._cffi_session is not None:
            await self._cffi_session.close()
            self._cffi_session = None

    async def _http_get(self, url: str, **kwargs: object) -> object:  # type: ignore[override]
        """curl_cffi-based GET with Chrome120 impersonation.

        Returns curl_cffi Response (compatible API: .status_code, .text).
        Caller must check status_code; no automatic retry (BaseScraper.retry
        decorator is per-method and not inherited cleanly when overridden).
        """
        if self._cffi_session is None:
            raise RuntimeError("YandexRealtyScraper must be used as async context manager")
        kwargs.setdefault("timeout", 30)
        return await self._cffi_session.get(url, **kwargs)

    async def fetch_around(
        self,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        page: int = 0,
        rooms: str | None = None,
        price_min: int | None = None,
        price_max: int | None = None,
    ) -> list[ScrapedLot]:
        """Fetch ONE page of SERP cards.

        lat/lon/radius_m ignored (Yandex uses path-based vtorichka URL);
        kept for BaseScraper compat + logging.
        rooms: room-bucket slug (one of ROOM_PATH keys) or None = all rooms.
        price_min/price_max: price filters in rubles (None = open-ended).
        """
        url = self._build_url(page=page, rooms=rooms, price_min=price_min, price_max=price_max)
        try:
            response = await self._http_get(url)
        except Exception:
            logger.exception("yandex serp fetch failed: %s", url)
            return []
        if response.status_code != 200:
            logger.warning("yandex serp returned %d for %s", response.status_code, url)
            return []

        # NBSP → space: parse_rub() buggy on \xa0 in capture groups → price=None (T4)
        html = response.text.replace("\xa0", " ")

        if _is_captcha(html):
            logger.warning(
                "yandex serp: captcha detected on page=%d url=%s — returning empty",
                page,
                url,
            )
            return []

        lots = self._parse_html(html, page=page)
        logger.info(
            "yandex serp page=%d city=%s rooms=%s price=[%s-%s]: %d cards",
            page,
            self.city,
            rooms or "all",
            price_min,
            price_max,
            len(lots),
        )
        await self.sleep_between_requests()
        return lots

    async def fetch_around_multi_room(
        self,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = MAX_PAGES,
        rooms_list: list[str] | None = None,
        price_ranges: list[tuple[int | None, int | None]] | None = None,
        **_legacy_kwargs: Any,  # swallow old callers' kwargs
    ) -> list[ScrapedLot]:
        """Fetch SERP via rooms × price-range combos to bypass SERP ~575-card cap.

        When rooms_list/price_ranges are provided (T5 combos mode), iterates all
        (room, price_min, price_max) combinations, each up to max_pages pages,
        deduplicating by source_id/source_url across combos.

        When both are None (legacy mode), falls back to single citywide sweep
        (original behaviour — up to max_pages pages, no filtering).
        """
        seen: dict[str, ScrapedLot] = {}

        if rooms_list is None and price_ranges is None:
            # Legacy / citywide mode — no combos
            combos: list[tuple[str | None, int | None, int | None]] = [(None, None, None)]
        else:
            r_list = rooms_list or list(ROOM_PATH.keys())
            p_ranges = price_ranges or DEFAULT_PRICE_RANGES
            combos = [(r, lo, hi) for r in r_list for lo, hi in p_ranges]

        for rooms, price_min, price_max in combos:
            combo_label = _combo_label(rooms, price_min, price_max)
            for page in range(max_pages):
                lots = await self.fetch_around(
                    lat,
                    lon,
                    radius_m,
                    page=page,
                    rooms=rooms,
                    price_min=price_min,
                    price_max=price_max,
                )
                if not lots:
                    logger.debug(
                        "yandex combos [%s] page=%d: empty — stopping combo",
                        combo_label,
                        page,
                    )
                    break
                for lot in lots:
                    key = lot.source_id or lot.source_url
                    if key and key not in seen:
                        seen[key] = lot

        logger.info(
            "yandex serp aggregate: %d unique lots (city=%s, %d combos)",
            len(seen),
            self.city,
            len(combos),
        )
        return list(seen.values())

    def _build_url(
        self,
        page: int = 0,
        rooms: str | None = None,
        price_min: int | None = None,
        price_max: int | None = None,
    ) -> str:
        """Build Yandex SERP URL.

        rooms: key from ROOM_PATH for path-based room bucket; None = all rooms.
        price_min/price_max: optional price filters in rubles.
        newFlat=NO_DEAL is added when any price filter is set — prevents Yandex
        from canonicalizing away /vtorichniy-rynok/ for some room slugs.
        """
        if rooms and rooms in ROOM_PATH:
            path = f"/{self.city}/kupit/kvartira/{ROOM_PATH[rooms]}/vtorichniy-rynok/"
        else:
            path = f"/{self.city}/kupit/kvartira/vtorichniy-rynok/"
        base = f"{self.base_url}{path}"

        params: dict[str, str | int] = {}
        if price_min is not None:
            params["priceMin"] = price_min
        if price_max is not None:
            params["priceMax"] = price_max
        if price_min is not None or price_max is not None:
            # Guarantee vtorichka — Yandex sometimes drops /vtorichniy-rynok/ on canonicalize
            params["newFlat"] = "NO_DEAL"
        if page > 0:
            params["page"] = page

        if params:
            return f"{base}?{urlencode(params)}"
        return base

    def _parse_html(self, html: str, page: int = 0) -> list[ScrapedLot]:
        # NBSP-fix applied by caller (fetch_around) before _parse_html;
        # also applied here as a safety net for direct callers (tests, scripts).
        html = html.replace("\xa0", " ")

        # Extract offer-level area from embedded JSON state first (covers ~82% of
        # cards where DOM text is empty pre-hydration). Falls back to DOM regex per card.
        state_areas = _extract_offer_areas_from_state(html)
        if state_areas:
            logger.debug("yandex serp: state areas extracted for %d offers", len(state_areas))
        tree = HTMLParser(html)
        cards = tree.css('[data-test="OffersSerpItem"]')
        lots: list[ScrapedLot] = []
        for card in cards:
            lot = self._card_to_lot(card, page=page, state_areas=state_areas)
            if lot is not None:
                lots.append(lot)
        return lots

    def _card_to_lot(
        self,
        card: Node,
        page: int = 0,
        state_areas: dict[str, float] | None = None,
    ) -> ScrapedLot | None:
        try:
            # offer_id — required
            offer_link = card.css_first('a[href^="/offer/"]')
            if offer_link is None:
                return None
            href = offer_link.attributes.get("href", "")
            id_match = RE_OFFER_ID.search(href)
            if not id_match:
                return None
            offer_id = id_match.group(1)

            text = card.text(strip=True)

            # Price — required (ScrapedLot.price_rub > 0)
            price_m = RE_PRICE.search(text)
            price_rub = parse_rub(price_m.group(1)) if price_m else None
            if not price_rub or price_rub <= 0:
                return None

            # Area: prefer JSON state (covers pre-hydration DOM where text is empty),
            # fall back to DOM regex.
            area_m2: float | None = (state_areas or {}).get(offer_id)
            if area_m2 is None:
                area_m = RE_TITLE_AREA.search(text)
                area_m2 = float(area_m.group(1).replace(",", ".")) if area_m else None

            rooms = self._parse_rooms(text)

            floor_m = RE_FLOOR.search(text)
            floor = int(floor_m.group(1)) if floor_m else None
            total_floors = int(floor_m.group(2)) if floor_m else None

            ppm2_m = RE_PPM2.search(text)
            price_per_m2 = parse_rub(ppm2_m.group(1)) if ppm2_m else None

            bargain = "торг" in text.lower()
            listing_date = parse_listing_date(text)

            # Repair state — инференс из текста карточки (#622). Yandex SERP не
            # отдаёт структурного поля ремонта; берём сигнал из текста сниппета.
            repair_state = infer_repair_state_from_text(text)

            # Address — prefer the full address container (street + house number);
            # fall back to the street aggregator link (street name only) if absent.
            # See _extract_address for details.
            address = self._extract_address(card)

            # Photos
            photo_urls = self._extract_photos(card)

            # Newbuilding linkage
            house_source: str | None = None
            house_ext_id: str | None = None
            house_url: str | None = None
            nb_link = card.css_first('a[href*="/kupit/novostrojka/"]')
            if nb_link is not None:
                nb_href = nb_link.attributes.get("href", "")
                nb_match = RE_JK_ID.search(nb_href)
                if nb_match:
                    house_source = "yandex_realty_nb"
                    house_ext_id = nb_match.group(2)
                    house_url = (
                        nb_href
                        if nb_href.startswith("http")
                        else f"https://realty.yandex.ru{nb_href}"
                    )

            return ScrapedLot(
                source=self.name,
                source_url=f"https://realty.yandex.ru/offer/{offer_id}/",
                source_id=offer_id,
                address=address,
                lat=None,
                lon=None,
                rooms=rooms,
                area_m2=area_m2,
                floor=floor,
                total_floors=total_floors,
                repair_state=repair_state,
                price_rub=price_rub,
                price_per_m2=price_per_m2,
                bargain_allowed=bargain,
                listing_date=listing_date,
                photo_urls=photo_urls,
                house_source=house_source,
                house_ext_id=house_ext_id,
                house_url=house_url,
                listing_segment="vtorichka",
                raw_payload={
                    "card_text": text[:1000],
                    "page": page,
                    "city": self.city,
                },
            )
        except Exception:
            logger.exception("yandex card parse failed (page=%d)", page)
            return None

    @staticmethod
    def _extract_address(card: Node) -> str | None:
        """Extract full address (street + house number) from a SERP card.

        Yandex renders the address inside a container like:
            <div class="AddressWithGeoLinks__addressContainer--XXXX">
              <a href="/.../kupit/kvartira/st-...">улица Энгельса</a>, 38
            </div>

        The street-only `<a>` tag was used historically — it gave just the
        street name and broke forward-geocoding precision (street-level only,
        not exact house). Reading the container's text yields the full
        "<street>, <house>" pair, including letter/fraction suffixes
        (e.g. "2В", "2/2", "14к2"). Some cards prefix district/city info
        (e.g. "Берёзовский, Александровский проспект, 5А") — that's still
        a valid, more specific address for the geocoder.

        Falls back to the street-only link when the container is missing
        (older layout / unexpected DOM) so we never regress to NULL.
        """
        addr_div = card.css_first(ADDRESS_CONTAINER_SELECTOR)
        if addr_div is not None:
            # text(strip=False) preserves spaces between text nodes and
            # the anchor's text (selectolax with strip=True collapses them
            # and yields "улица Энгельса,38" instead of "улица Энгельса, 38").
            raw = addr_div.text(strip=False) or ""
            normalized = _RE_WS.sub(" ", raw).strip().strip(",").strip()
            if normalized:
                return normalized
        street_link = card.css_first('a[href*="/kupit/kvartira/st-"]')
        return street_link.text(strip=True) if street_link else None

    @staticmethod
    def _parse_rooms(text: str) -> int | None:
        m = RE_TITLE_ROOMS.search(text)
        if not m:
            return None
        if m.group(2):  # studio
            return 0
        if m.group(1):  # numbered
            try:
                return int(m.group(1))
            except ValueError:
                return None
        return None

    @staticmethod
    def _extract_photos(card: Node) -> list[str]:
        urls: list[str] = []
        for img in card.css("img"):
            src = img.attributes.get("src", "") or ""
            if PHOTO_DOMAIN in src:
                urls.append(src.replace(PHOTO_SIZE_FROM, PHOTO_SIZE_TO))
            if len(urls) >= 5:
                break
        return urls


def _combo_label(
    rooms: str | None,
    price_min: int | None,
    price_max: int | None,
) -> str:
    """Human-readable label for a (rooms, price_min, price_max) combo."""
    r = rooms or "all-rooms"
    if price_min is None and price_max is None:
        return f"{r}/any-price"
    lo = f"{price_min // 1_000_000}M" if price_min is not None else "0"
    hi = f"{price_max // 1_000_000}M" if price_max is not None else "inf"
    return f"{r}/{lo}-{hi}"
