"""Yandex.Nedvizhimost scraper (realty.yandex.ru) -- gate-API JSON parser.

Transport: BrowserFetcher (camoufox real-browser fingerprint via tradein-browser service).
camoufox's fingerprint is not tarpitted by yandex and does not burn the mobile-proxy IP;
raw system-curl over SOCKS5 gets the proxy IP flagged. One BrowserFetcher context is opened
per scraper session (__aenter__) and reused across all page fetches.

camoufox returns the gate-API JSON wrapped in HTML (verified prod 2026-06-17):
    <html><head>...</head><body><pre>{"response":...}</pre></body></html>
JSON is extracted via _extract_json_from_content() (handles the <pre> wrapper).

API path: GET https://realty.yandex.ru/gate/react-page/get/
  _pageType=search&_providers=react-search-results-data
  type=SELL&category=APARTMENT&newFlat=NO&rgid=559132 (EKB)
  roomsTotal=<int|STUDIO>&priceMin=...&priceMax=...&page=<1-based>

Response shape: response.search.offers.{entities[], pager{page,pageSize,totalItems,totalPages}}

Page param is 1-based: page=1 -> first page (pager.page=0 in response).
page=0 -> API returns error.  Paginate from page=1 to totalPages (inclusive).

Studio roomsTotal:
  Request: roomsTotal=STUDIO
  Response entities: roomsTotal=None -> mapped to rooms=0 (existing convention).

Strangler-копия `app.services.scrapers.yandex_realty` (#2133). Развязка от `app.*`:
  - `app.core.config.settings` → инжектируемый `ScraperConfig` (yandex_proxy_rotate_url,
    avito_proxy_rotate_url)
  - `app.services.scraper_settings.get_scraper_delay` → инжектируемый `delay_provider`
  - `app.services.scrapers.{base,browser_fetcher,price_brackets}` → `scraper_kit.*`
  - `app.services.scrapers.house_type_normalizer` → `scraper_kit.providers.yandex.shared`
"""

from __future__ import annotations

import asyncio
import json
import logging
import math
import re
from collections.abc import Callable
from dataclasses import dataclass
from datetime import UTC, date, datetime
from typing import TYPE_CHECKING, Any
from urllib.parse import urlencode

from curl_cffi.requests import AsyncSession as _CurlCffiSession

from scraper_kit.base import BaseScraper, ScrapedLot
from scraper_kit.browser_fetcher import BrowserFetcher
from scraper_kit.house_type_normalizer import normalize_house_type
from scraper_kit.price_brackets import get_price_seed_brackets
from scraper_kit.pricing import BisectionConfig, ProbeFailPolicy, ProbeResult, walk_price_range
from scraper_kit.providers._base import build_browser_fetcher

if TYPE_CHECKING:
    from scraper_kit.contracts import ProxyProvider, ScraperConfig

logger = logging.getLogger(__name__)

# camoufox renders JSON endpoints as <pre>{"response":...}</pre> inside an HTML wrapper.
_RE_PRE_JSON = re.compile("<pre[^>]*>(\\{.*?)</pre>", re.DOTALL | re.IGNORECASE)

DEFAULT_CITY = "ekaterinburg"
_GATE_MAX_PAGES_CAP = 50
_EKB_RGID = 559132
_GATE_URL = "https://realty.yandex.ru/gate/react-page/get/"

ROOM_GATE_PARAM: dict[str, str | int] = {
    "studio": "STUDIO",
    "1": 1,
    "2": 2,
    "3": 3,
    "4+": 4,
}

ROOM_PATH: dict[str, str] = {
    "studio": "studiya",
    "1": "odnokomnatnaya",
    "2": "dvuhkomnatnaya",
    "3": "trehkomnatnaya",
    "4+": "4-i-bolee",
}

DEFAULT_PRICE_RANGES: list[tuple[int | None, int | None]] = [
    (None, 5_000_000),
    (5_000_000, 7_000_000),
    (7_000_000, 10_000_000),
    (10_000_000, 15_000_000),
    (15_000_000, 25_000_000),
    (25_000_000, None),
]

_YANDEX_MAX_PRICE = 200_000_000
# ВНИМАНИЕ: реальный порог split у yandex — 500_000 (см. _YANDEX_SPLIT_MIN_BRACKET),
# а НЕ 50_000. Историческая _YANDEX_MIN_BRACKET=50_000 никогда не использовалась в
# _walk_price_range (там был локальный `_min_bracket = 500_000`), оставлена для
# обратной совместимости импортов; движок бисекции использует 500_000.
_YANDEX_MIN_BRACKET = 50_000
# Минимальный размах диапазона для деления (yandex делит только пока (hi-lo) >= 500k)
# и жёсткий предел глубины рекурсии — переносим в единый движок как есть.
_YANDEX_SPLIT_MIN_BRACKET = 500_000
_YANDEX_MAX_SPLIT_DEPTH = 8
_GATE_PAGE_SIZE = 20
_YANDEX_TARPIT_MAX_RETRIES: int = 2

# Таймаут httpx-клиента BrowserFetcher для yandex-пути: 30s вместо глобального 120s.
# Один проблемный combo занимает ≤30s×(1+_YANDEX_TARPIT_MAX_RETRIES)=90s вместо 360s.
_YANDEX_BROWSER_FETCH_TIMEOUT_S: float = 30.0


@dataclass
class _CurlResponse:
    status_code: int
    text: str


def _is_gate_error(payload: dict[str, Any]) -> bool:
    """Return True when gate-API returned an error object instead of search results."""
    return "error" in payload or "response" not in payload


def _extract_gate_data(
    payload: dict[str, Any],
) -> tuple[list[dict[str, Any]], dict[str, Any]] | None:
    """Extract (entities, pager) from valid gate-API response. None on missing path."""
    try:
        offers = payload["response"]["search"]["offers"]
        entities: list[dict[str, Any]] = offers["entities"]
        pager: dict[str, Any] = offers["pager"]
        return entities, pager
    except (KeyError, TypeError):
        return None


def _extract_json_from_content(content: str) -> str | None:
    """Extract raw JSON text from camoufox HTML-wrapped content.

    camoufox renders a JSON endpoint as:
        <html><head>...</head><body><pre>{"response":...}</pre></body></html>

    Strategy:
    1. Try <pre>...</pre> regex (primary -- matches prod output).
    2. Fallback: slice from the first '{' (handles raw-JSON response, no wrapper).

    Returns the JSON string, or None if no '{' is found.
    """
    m = _RE_PRE_JSON.search(content)
    if m:
        return m.group(1)

    idx = content.find("{")
    if idx != -1:
        return content[idx:]

    return None


def _segment_for(new_flat: str) -> str:
    """Map a newFlat gate-API value to the listings.listing_segment value."""
    return "novostroyki" if new_flat == "YES" else "vtorichka"


def _parse_creation_date(raw: Any) -> tuple[date | None, int | None]:
    """Parse gate-API creationDate ('2026-05-24T15:10:27Z') → (date, days_on_market).

    days_on_market = floor((now - creationDate).days), clamped to >= 0.
    Returns (None, None) when raw is absent or unparseable (never raises).
    """
    if not raw or not isinstance(raw, str):
        return None, None
    txt = raw.strip()
    # datetime.fromisoformat handles 'Z' only from py3.11+, but normalise defensively.
    if txt.endswith("Z"):
        txt = txt[:-1] + "+00:00"
    try:
        dt = datetime.fromisoformat(txt)
    except ValueError:
        return None, None
    if dt.tzinfo is None:
        dt = dt.replace(tzinfo=UTC)
    listing_date = dt.date()
    delta_days = (datetime.now(UTC) - dt).days
    days_on_market = max(delta_days, 0)
    return listing_date, days_on_market


def _author_fields(author: dict[str, Any]) -> tuple[bool | None, bool | None, str | None]:
    """Map entity.author → (is_homeowner, is_pro_seller, agency_name).

    author.category ∈ {OWNER, AGENCY, AGENT, ...}.
      OWNER          → is_homeowner=True
      AGENCY / AGENT → is_pro_seller=True, agency_name = organization or agentName
    Returns (None, None, None) when author/category absent.
    """
    category = (author.get("category") or "").upper()
    if not category:
        return None, None, None
    if category == "OWNER":
        return True, None, None
    if category in {"AGENCY", "AGENT"}:
        agency_name = author.get("organization") or author.get("agentName") or None
        return None, True, agency_name
    return None, None, None


def _to_bigint(raw: Any) -> int | None:
    """Coerce a gate-API numeric value (str or number) to int. None when unparseable."""
    if raw is None:
        return None
    try:
        return int(float(raw))
    except (TypeError, ValueError):
        return None


def _predicted_prices(entity: dict[str, Any]) -> tuple[int | None, int | None, int | None]:
    """Extract (value, min, max) from entity.predictions.predictedPrice.

    Yandex's own per-offer valuation. The gate-API nests it under
    ``predictions.predictedPrice`` (NOT ``predictedPrice.predictedPrice`` —
    that key does not exist; verified live 2026-06-18 against the gate-API).
    Fields are strings (e.g. "7554000"). Returns (None, None, None) when the
    nested path is absent (e.g. novostroyki offers carry no valuation).
    """
    outer = entity.get("predictions") or {}
    inner = outer.get("predictedPrice") or {}
    if not isinstance(inner, dict):
        return None, None, None
    return (
        _to_bigint(inner.get("value")),
        _to_bigint(inner.get("min")),
        _to_bigint(inner.get("max")),
    )


def _metro_stations(location: dict[str, Any]) -> list[dict[str, Any]]:
    """Build metro_stations [{name, min}] from location.metroList[] (+ location.metro)."""
    stations: list[dict[str, Any]] = []
    raw_list = location.get("metroList")
    if isinstance(raw_list, list):
        for m in raw_list:
            if isinstance(m, dict) and m.get("name"):
                stations.append({"name": m.get("name"), "min": m.get("timeToMetro")})
    if not stations:
        single = location.get("metro")
        if isinstance(single, dict) and single.get("name"):
            stations.append({"name": single.get("name"), "min": single.get("timeToMetro")})
    return stations


def _parse_gate_json(
    payload: dict[str, Any], page_param: int = 1, new_flat: str = "NO"
) -> list[ScrapedLot]:
    """Parse gate-API JSON payload into a list of ScrapedLot.

    Verified field mapping (from .issdump/yandex_gate_api_sample.json + live prod 2026-06-17):
      offerId                         -> source_id
      url (//realty.yandex.ru/offer/) -> source_url (canonical per-offer, #2235:
                                         partner/redirect url -> offer_id-based)
      price.value                     -> price_rub
      price.valuePerPart              -> price_per_m2
      area.value                      -> area_m2
      livingSpace.value               -> living_area_m2
      kitchenSpace.value              -> kitchen_area_m2 (+ raw_payload["kitchen_area_m2"])
      roomsTotal (int | None)         -> rooms (None -> 0 for studio)
      floorsOffered[0]                -> floor
      floorsTotal                     -> total_floors
      building.builtYear              -> year_built
      building.buildingType           -> house_type (normalize_house_type: MONOLIT->monolith)
      ceilingHeight                   -> ceiling_height_m (+ raw_payload["ceiling_height"])
      location.point.latitude/longitude -> lat / lon
      location.geocoderAddress        -> address (full, with house number)
      mainImages[]                    -> photo_urls (prepend "https:", up to 5)
      building.siteId                 -> house_ext_id (JK linkage)
      listing_segment                 -> "novostroyki" (newFlat=YES) | "vtorichka" (newFlat=NO)
      creationDate (ISO Z)            -> listing_date / publish_date + days_on_market
      description                     -> description
      author.category/organization   -> is_homeowner / is_pro_seller / agency_name
      predictedPrice.predictedPrice.* -> predicted_price_rub / _min / _max
      price.trend / price.previous    -> price_trend / price_previous_rub
      location.metroList[]            -> metro_stations [{name, min}]
      offerId                         -> yandex_offer_id
    """
    result = _extract_gate_data(payload)
    if result is None:
        logger.warning("yandex gate: response missing response.search.offers path")
        return []

    entities, _pager = result
    lots: list[ScrapedLot] = []
    for entity in entities:
        lot = _entity_to_lot(entity, page_param=page_param, new_flat=new_flat)
        if lot is not None:
            lots.append(lot)
    return lots


def _canonical_source_url(url_raw: str, offer_id: str) -> str:
    """Каноничный per-offer source_url для yandex-оффера (#2235).

    gate-API `url` для вторички = `//realty.yandex.ru/offer/<id>` (уникален на
    оффер). Для новостроек/partner-офферов там НЕканоническая ссылка на сайт
    застройщика или рекламный редирект (na100.pro/go.php, atlasdevelopment.su, …),
    которую делят МНОГО разных квартир одного ЖК → source_url перестаёт быть
    уникальным (204 «дубля» среди is_active). Возвращаем стабильный уникальный
    URL из offer_id, если исходный не ведёт на realty.yandex; иначе — исходный.
    """
    source_url = f"https:{url_raw}" if url_raw.startswith("//") else url_raw
    is_yandex_realty = "realty.yandex" in source_url or "realty.ya.ru" in source_url
    if not source_url or not is_yandex_realty:
        return f"https://realty.yandex.ru/offer/{offer_id}/"
    return source_url


def _entity_to_lot(
    entity: dict[str, Any], page_param: int = 1, new_flat: str = "NO"
) -> ScrapedLot | None:
    """Convert one gate-API entity dict to ScrapedLot. None on missing required fields."""
    try:
        offer_id = str(entity.get("offerId") or "")
        if not offer_id:
            return None

        # #2235: канонизируем source_url — partner/redirect url новостроек делят
        # много офферов одного ЖК → дубли source_url. См. _canonical_source_url.
        source_url = _canonical_source_url(entity.get("url") or "", offer_id)

        price_dict = entity.get("price") or {}
        price_val = price_dict.get("value")
        if not price_val or price_val <= 0:
            return None
        price_rub = int(price_val)

        price_per_m2_raw = price_dict.get("valuePerPart")
        price_per_m2 = int(price_per_m2_raw) if price_per_m2_raw else None

        area_dict = entity.get("area") or {}
        area_m2_raw = area_dict.get("value")
        area_m2 = float(area_m2_raw) if area_m2_raw is not None else None

        living_dict = entity.get("livingSpace") or {}
        living_area_raw = living_dict.get("value")
        living_area_m2 = float(living_area_raw) if living_area_raw is not None else None

        kitchen_dict = entity.get("kitchenSpace") or {}
        kitchen_area_raw = kitchen_dict.get("value")
        kitchen_area_m2 = float(kitchen_area_raw) if kitchen_area_raw is not None else None

        rooms_raw = entity.get("roomsTotal")
        if rooms_raw is None:
            rooms = 0  # studio
        else:
            try:
                rooms = int(rooms_raw)
            except (TypeError, ValueError):
                rooms = None

        floors_offered = entity.get("floorsOffered") or []
        floor = int(floors_offered[0]) if floors_offered else None
        floors_total_raw = entity.get("floorsTotal")
        total_floors = int(floors_total_raw) if floors_total_raw is not None else None

        ceiling_height = entity.get("ceilingHeight")
        # Колонка ceiling_height — numeric(3,2), max 9.99. Yandex SERP отдаёт мусор
        # (видели 18 м) → запись out-of-range уронила бы весь батч DataError'ом
        # (per-lot SAVEPOINT ловит только IntegrityError). Берём только правдоподобный
        # диапазон 2.0–6.0 м, иначе None; сырое значение остаётся в raw_payload. (#2007)
        _ceiling_raw = float(ceiling_height) if ceiling_height is not None else None
        ceiling_height_m = (
            _ceiling_raw if _ceiling_raw is not None and 2.0 <= _ceiling_raw <= 6.0 else None
        )

        building = entity.get("building") or {}
        year_built_raw = building.get("builtYear")
        year_built = int(year_built_raw) if year_built_raw is not None else None
        # buildingType — SCREAMING (MONOLIT/BRICK/...). Нормализуем в канон, иначе
        # estimator soft-penalty штрафует yandex-аналоги (#2007).
        raw_building_type = building.get("buildingType")
        house_type = normalize_house_type(raw_building_type)

        site_id = building.get("siteId")
        site_name = building.get("siteName")
        house_source: str | None = None
        house_ext_id: str | None = None
        if site_id:
            house_source = "yandex_realty_nb"
            house_ext_id = str(site_id)

        location = entity.get("location") or {}
        point = location.get("point") or {}
        lat_raw = point.get("latitude")
        lon_raw = point.get("longitude")
        lat = float(lat_raw) if lat_raw is not None else None
        lon = float(lon_raw) if lon_raw is not None else None

        address = location.get("geocoderAddress") or location.get("streetAddress") or None

        # ── Rich gate-API fields ───────────────────────────────────────────
        listing_date, days_on_market = _parse_creation_date(entity.get("creationDate"))
        description = entity.get("description") or None

        author = entity.get("author") or {}
        is_homeowner, is_pro_seller, agency_name = _author_fields(author)

        predicted_value, predicted_min, predicted_max = _predicted_prices(entity)

        price_trend = price_dict.get("trend") or None
        price_previous_rub = _to_bigint(price_dict.get("previous"))

        metro_stations = _metro_stations(location)

        photo_urls: list[str] = []
        for img in entity.get("mainImages") or []:
            if img:
                photo_urls.append(f"https:{img}" if img.startswith("//") else img)
            if len(photo_urls) >= 5:
                break

        return ScrapedLot(
            source="yandex",
            source_url=source_url,
            source_id=offer_id,
            address=address,
            lat=lat,
            lon=lon,
            rooms=rooms,
            area_m2=area_m2,
            living_area_m2=living_area_m2,
            floor=floor,
            total_floors=total_floors,
            year_built=year_built,
            house_type=house_type,
            kitchen_area_m2=kitchen_area_m2,
            ceiling_height_m=ceiling_height_m,
            price_rub=price_rub,
            price_per_m2=price_per_m2,
            photo_urls=photo_urls,
            house_source=house_source,
            house_ext_id=house_ext_id,
            house_url=None,
            listing_segment=_segment_for(new_flat),
            listing_date=listing_date,
            publish_date=listing_date,
            days_on_market=days_on_market,
            description=description,
            is_homeowner=is_homeowner,
            is_pro_seller=is_pro_seller,
            agency_name=agency_name,
            metro_stations=metro_stations,
            yandex_offer_id=offer_id,
            predicted_price_rub=predicted_value,
            predicted_price_min=predicted_min,
            predicted_price_max=predicted_max,
            price_trend=price_trend,
            price_previous_rub=price_previous_rub,
            raw_payload={
                "page_param": page_param,
                "ceiling_height": ceiling_height,
                "kitchen_area_m2": kitchen_area_raw,
                "raw_building_type": raw_building_type,  # сырой SCREAMING — для отладки
                "site_name": site_name,
                "offer_id": offer_id,
            },
        )
    except Exception:
        logger.exception("yandex gate: entity parse failed offer_id=%s", entity.get("offerId"))
        return None


class YandexRealtyScraper(BaseScraper):
    name = "yandex"
    base_url = "https://realty.yandex.ru"
    request_delay_sec = 5.0

    def __init__(
        self,
        config: ScraperConfig,
        city: str = DEFAULT_CITY,
        *,
        delay_provider: Callable[[str], float] | None = None,
        proxy_provider: ProxyProvider | None = None,
        city_rgid: int | None = None,
    ) -> None:
        super().__init__()
        self.city = city
        # Strangler-инжекция (#2133): конфиг и провайдер задержки приходят снаружи
        # вместо прямого импорта app.core.config.settings /
        # app.services.scraper_settings.get_scraper_delay. Продуктовая сторона
        # передаёт RealScraperConfig + get_scraper_delay; kit не знает про app / БД.
        self._config = config
        if delay_provider is not None:
            self.request_delay_sec = delay_provider(self.name)
        # #2160: пул прокси для camoufox-браузера (за флагом use_proxy_pool_browser).
        # None → BrowserFetcher без пула = env-прокси браузера (ship-dark, как сейчас).
        self._proxy_provider = proxy_provider
        self._browser: BrowserFetcher | None = None
        # #12 (oblast rollout): rgid города-цели SERP-запроса (gate-API). None →
        # ЕКБ-дефолт (_EKB_RGID).
        self._rgid = city_rgid or _EKB_RGID
        # _cffi_session retained only for _rotate_ip (changeip call).
        self._cffi_session: _CurlCffiSession | None = None
        self._cookies: dict[str, str] = {}

    async def __aenter__(self) -> YandexRealtyScraper:  # type: ignore[override]
        """Open ONE BrowserFetcher (camoufox) session, reused across all page fetches.

        camoufox's real-browser fingerprint avoids the mobile-proxy IP tarpit
        escalation that raw system-curl over SOCKS5 triggers. Opening per-page is
        expensive and unnecessary.

        fetch_timeout_s=_YANDEX_BROWSER_FETCH_TIMEOUT_S (30s) ограничивает потери
        на один тайм-аутовый запрос: worst-case 30s×(1+retries)=90s на combo вместо 360s.

        Собрано через shared `build_browser_fetcher()` (#2358 Foundation, #2363
        миграция) — endpoint/use_pool берутся из `config` mandatory, footgun-класс
        #2322/#2330 (пропущенный endpoint/use_pool) закрыт структурно.
        """
        self._browser = build_browser_fetcher(
            self._config,
            source="yandex",
            proxy_provider=self._proxy_provider,
            fetch_timeout_s=_YANDEX_BROWSER_FETCH_TIMEOUT_S,
        )
        await self._browser.__aenter__()
        return self

    async def __aexit__(self, *args: object) -> None:  # type: ignore[override]
        if self._browser is not None:
            await self._browser.__aexit__(*args)
            self._browser = None
        if self._cffi_session is not None:
            await self._cffi_session.close()
            self._cffi_session = None

    async def _http_get(self, url: str, **kwargs: object) -> _CurlResponse:  # type: ignore[override]
        """Fetch gate-API URL via the shared BrowserFetcher (camoufox).

        camoufox returns HTML-wrapped JSON; the JSON is extracted and returned as
        the body of a _CurlResponse so existing callers (json.loads(resp.text),
        resp.status_code checks) keep working unchanged.

        Maps outcomes onto the curl-style status contract:
          - JSON extracted  -> status_code=200, text=<json>
          - fetch raised / no JSON found -> status_code=0 (treated as transient
            failure by the caller retry logic)
        """
        if self._browser is None:
            raise RuntimeError("YandexRealtyScraper must be used as async context manager")

        try:
            content = await self._browser.fetch(url)
        except Exception:
            logger.warning("yandex gate: BrowserFetcher.fetch failed url=%s", url, exc_info=True)
            return _CurlResponse(status_code=0, text="")

        json_text = _extract_json_from_content(content)
        if not json_text:
            logger.warning(
                "yandex gate: could not extract JSON from content url=%s first200=%r",
                url,
                content[:200],
            )
            return _CurlResponse(status_code=0, text="")

        return _CurlResponse(status_code=200, text=json_text)

    async def _rotate_ip(self) -> bool:
        """Rotate mobile proxy IP via changeip URL. Returns True on success."""
        rotate_url = self._config.yandex_proxy_rotate_url or self._config.avito_proxy_rotate_url
        if not rotate_url:
            return False
        sep = "&" if "?" in rotate_url else "?"
        try:
            async with _CurlCffiSession(timeout=30) as rot:
                await rot.get(f"{rotate_url}{sep}format=json")
            await asyncio.sleep(9)
            logger.info("yandex proxy: IP rotated via changeip")
            return True
        except Exception:
            logger.warning("yandex proxy: IP rotation failed", exc_info=True)
            return False

    def _build_url(
        self,
        page: int = 1,
        rooms: str | None = None,
        price_min: int | None = None,
        price_max: int | None = None,
        new_flat: str = "NO",
    ) -> str:
        """Build gate-API URL. page: 1-based (page=0 -> API error).

        new_flat: "NO" -> vtorichka (default, preserves legacy behavior),
                  "YES" -> novostroyki (primary-sale / reassignment segment).
        """
        params: dict[str, str | int] = {
            "rgid": self._rgid,
            "type": "SELL",
            "category": "APARTMENT",
            "newFlat": new_flat,
            "_pageType": "search",
            "_providers": "react-search-results-data",
            "page": page,
        }
        if rooms and rooms in ROOM_GATE_PARAM:
            params["roomsTotal"] = ROOM_GATE_PARAM[rooms]
        if price_min is not None:
            params["priceMin"] = price_min
        if price_max is not None:
            params["priceMax"] = price_max
        return f"{_GATE_URL}?{urlencode(params)}"

    async def _fetch_page_json(
        self,
        rooms: str | None,
        page: int,
        price_min: int | None,
        price_max: int | None,
        new_flat: str = "NO",
    ) -> dict[str, Any] | None:
        """Fetch one gate-API page. Returns parsed JSON dict or None on error/invalid."""
        url = self._build_url(
            page=page, rooms=rooms, price_min=price_min, price_max=price_max, new_flat=new_flat
        )
        try:
            resp = await self._http_get(url, timeout=60)
        except Exception:
            logger.exception("yandex gate: GET failed url=%s", url)
            return None
        if resp.status_code != 200:  # type: ignore[union-attr]
            logger.warning("yandex gate: HTTP %d url=%s", resp.status_code, url)  # type: ignore[union-attr]
            return None
        try:
            payload: dict[str, Any] = json.loads(resp.text)  # type: ignore[union-attr]
        except (json.JSONDecodeError, ValueError):
            logger.warning("yandex gate: JSON parse failed url=%s", url)
            return None
        if _is_gate_error(payload):
            logger.warning("yandex gate: error response url=%s keys=%s", url, list(payload.keys()))
            return None
        return payload

    async def fetch_around(
        self,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        page: int = 1,
        rooms: str | None = None,
        price_min: int | None = None,
        price_max: int | None = None,
        new_flat: str = "NO",
    ) -> list[ScrapedLot]:
        """Fetch ONE page of gate-API results. lat/lon/radius_m ignored (uses rgid,
        city-scoped — self._rgid, ЕКБ-дефолт если city_rgid не задан).

        page: 1-based. Tarpit resilience: status_code==0 or JSON error
        -> rotate IP + retry up to _YANDEX_TARPIT_MAX_RETRIES times.
        Non-tarpit failures (4xx/5xx) are not retried.
        new_flat: "NO" -> vtorichka, "YES" -> novostroyki.
        """
        url = self._build_url(
            page=page, rooms=rooms, price_min=price_min, price_max=price_max, new_flat=new_flat
        )

        payload: dict[str, Any] | None = None
        for attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES):
            try:
                response = await self._http_get(url, timeout=60)
            except Exception:
                logger.exception("yandex gate fetch failed: %s", url)
                return []

            status = response.status_code  # type: ignore[union-attr]

            if status == 0:
                logger.warning(
                    "yandex gate: status=0 (tarpit?) rooms=%s page=%d"
                    " -- rotating IP + retry attempt %d",
                    rooms,
                    page,
                    attempt + 1,
                )
                await self._rotate_ip()
                await asyncio.sleep(2)
                continue

            if status != 200:
                logger.warning(
                    "yandex gate: HTTP %d rooms=%s page=%d url=%s", status, rooms, page, url
                )
                return []

            try:
                payload = json.loads(response.text)  # type: ignore[union-attr]
            except (json.JSONDecodeError, ValueError):
                logger.warning(
                    "yandex gate: JSON parse failed rooms=%s page=%d -- rotating + retry %d",
                    rooms,
                    page,
                    attempt + 1,
                )
                await self._rotate_ip()
                await asyncio.sleep(2)
                continue

            if _is_gate_error(payload):
                logger.warning(
                    "yandex gate: error payload rooms=%s page=%d keys=%s -- rotating + retry %d",
                    rooms,
                    page,
                    list(payload.keys()),
                    attempt + 1,
                )
                await self._rotate_ip()
                await asyncio.sleep(2)
                payload = None
                continue

            break

        if payload is None:
            logger.warning(
                "yandex gate: all retries exhausted rooms=%s page=%d -- returning empty",
                rooms,
                page,
            )
            return []

        lots = _parse_gate_json(payload, page_param=page, new_flat=new_flat)
        logger.info(
            "yandex gate page=%d rooms=%s price=[%s-%s] newFlat=%s: %d cards",
            page,
            rooms or "all",
            price_min,
            price_max,
            new_flat,
            len(lots),
        )
        await self.sleep_between_requests()
        return lots

    async def fetch_around_multi_room(
        self,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = _GATE_MAX_PAGES_CAP,
        rooms_list: list[str] | None = None,
        price_ranges: list[tuple[int | None, int | None]] | None = None,
        segments: list[str] | None = None,
        on_combo: Any = None,
        **_legacy_kwargs: Any,
    ) -> list[ScrapedLot]:
        """Fetch via segment x rooms x price-range combos; paginate each combo to totalPages.

        Pagination driven by pager.totalPages from first page response.
        Each combo iterates page=1 .. min(totalPages, max_pages, _GATE_MAX_PAGES_CAP).
        Deduplicates by source_id/source_url across ALL combos AND segments (one `seen`).
        Legacy mode (rooms_list=None, price_ranges=None): single citywide sweep.
        segments: list of newFlat values, default ["NO"] (vtorichka only). Pass
            ["NO", "YES"] to sweep both vtorichka and novostroyki in one call.

        on_combo: опциональный callback(combo_label: str, new_lots: list[ScrapedLot]) -> None.
            Вызывается после каждого combo с новыми (не повторными) лотами из него.
            Позволяет инкрементальный save: вызывающий код сохраняет new_lots сразу,
            не дожидаясь конца sweep'а. Дубли между combo НЕ передаются повторно.
        """
        seen: dict[str, ScrapedLot] = {}
        _segments = segments or ["NO"]
        combos_skipped = 0

        if rooms_list is None and price_ranges is None:
            combos: list[tuple[str | None, int | None, int | None]] = [(None, None, None)]
        else:
            r_list = rooms_list or list(ROOM_PATH.keys())
            p_ranges = price_ranges or DEFAULT_PRICE_RANGES
            combos = [(r, lo, hi) for r in r_list for lo, hi in p_ranges]

        for new_flat in _segments:
            _seg = _segment_for(new_flat)
            for rooms, price_min, price_max in combos:
                combo_label = f"{_seg}/{_combo_label(rooms, price_min, price_max)}"
                total_pages: int | None = None
                combo_new_lots: list[ScrapedLot] = []
                combo_skipped = False

                for page in range(1, max_pages + 1):
                    if page == 1:
                        payload_p1: dict[str, Any] | None = None
                        p1_url = self._build_url(
                            page=1,
                            rooms=rooms,
                            price_min=price_min,
                            price_max=price_max,
                            new_flat=new_flat,
                        )
                        for _attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES):
                            try:
                                resp = await self._http_get(p1_url, timeout=60)
                            except Exception:
                                logger.exception(
                                    "yandex gate combo fetch failed combo=%s", combo_label
                                )
                                break
                            if resp.status_code == 0:  # type: ignore[union-attr]
                                logger.warning(
                                    "yandex gate: tarpit combo=%s page=1 attempt=%d -- rotating",
                                    combo_label,
                                    _attempt + 1,
                                )
                                await self._rotate_ip()
                                await asyncio.sleep(2)
                                continue
                            if resp.status_code != 200:  # type: ignore[union-attr]
                                logger.warning(
                                    "yandex gate: HTTP %d combo=%s page=1",
                                    resp.status_code,
                                    combo_label,  # type: ignore[union-attr]
                                )
                                break
                            try:
                                payload_p1 = json.loads(resp.text)  # type: ignore[union-attr]
                            except (json.JSONDecodeError, ValueError):
                                logger.warning(
                                    "yandex gate: JSON parse error combo=%s page=1", combo_label
                                )
                                payload_p1 = None
                                await self._rotate_ip()
                                await asyncio.sleep(2)
                                continue
                            if _is_gate_error(payload_p1):
                                logger.warning(
                                    "yandex gate: error payload combo=%s page=1", combo_label
                                )
                                payload_p1 = None
                                break
                            break

                        if payload_p1 is None:
                            logger.warning(
                                "yandex gate combo [%s] page=1: all retries exhausted -- skipping",
                                combo_label,
                            )
                            combo_skipped = True
                            combos_skipped += 1
                            break

                        result = _extract_gate_data(payload_p1)
                        if result is None:
                            combo_skipped = True
                            combos_skipped += 1
                            break
                        _entities_p1, pager_p1 = result
                        total_pages = min(
                            pager_p1.get("totalPages", 1),
                            max_pages,
                            _GATE_MAX_PAGES_CAP,
                        )
                        lots_p1 = _parse_gate_json(payload_p1, page_param=1, new_flat=new_flat)
                        if not lots_p1:
                            logger.debug(
                                "yandex gate combo [%s] page=1: empty -- stopping", combo_label
                            )
                            break
                        for lot in lots_p1:
                            key = lot.source_id or lot.source_url
                            if key and key not in seen:
                                seen[key] = lot
                                combo_new_lots.append(lot)
                        await self.sleep_between_requests()
                        if total_pages <= 1:
                            break
                        continue

                    if total_pages is not None and page > total_pages:
                        break

                    lots = await self.fetch_around(
                        lat,
                        lon,
                        radius_m,
                        page=page,
                        rooms=rooms,
                        price_min=price_min,
                        price_max=price_max,
                        new_flat=new_flat,
                    )
                    if not lots:
                        logger.debug(
                            "yandex gate combo [%s] page=%d: empty -- stopping",
                            combo_label,
                            page,
                        )
                        break
                    for lot in lots:
                        key = lot.source_id or lot.source_url
                        if key and key not in seen:
                            seen[key] = lot
                            combo_new_lots.append(lot)

                # Инкрементальный save: вызываем on_combo если есть новые лоты.
                # Скипнутые combo (combo_skipped=True) не вызывают on_combo.
                if on_combo is not None and combo_new_lots and not combo_skipped:
                    try:
                        on_combo(combo_label, combo_new_lots)
                    except Exception:
                        logger.exception(
                            "yandex gate: on_combo callback failed combo=%s", combo_label
                        )

        logger.info(
            "yandex gate aggregate: %d unique lots (%d combos x %d segments=%s, skipped=%d)",
            len(seen),
            len(combos),
            len(_segments),
            _segments,
            combos_skipped,
        )
        return list(seen.values())

    async def fetch_all_secondary(
        self,
        *,
        rooms_buckets: list[str] | None = None,
        price_cap_per_bucket: int = 500,
        max_pages_per_bucket: int = _GATE_MAX_PAGES_CAP,
        concurrency: int = 4,
        on_bucket: Any = None,
        on_progress: Any = None,
        skip_buckets: set[str] | None = None,
    ) -> list[ScrapedLot]:
        """Exhaustive EKB vtorichka load via KOMNATNOST x TSENA partitioning.

        Uses gate-API JSON (totalItems from pager) instead of DOM state extraction.
        Стартует с общей data-driven seed-сетки ценовых брекетов
        (get_price_seed_brackets() — shared EKB grid, общая с avito/cian) вместо
        одной корень-бисекции [0, _YANDEX_MAX_PRICE]. Закрытые seed-брекеты передают
        hi = br_hi - 1 (priceMax эксклюзивный сверху → нет пересечений), открытый
        (br_hi is None) — hi=None (без priceMax, весь хвост люкса). Дедуп по
        source_id в общем seen страхует на стыках.
        """
        _buckets = rooms_buckets if rooms_buckets is not None else list(ROOM_PATH.keys())
        seen: dict[str, ScrapedLot] = {}

        for rooms_key in _buckets:
            logger.info(
                "yandex gate exhaustive: starting rooms=%s (price_cap=%d concurrency=%d)",
                rooms_key,
                price_cap_per_bucket,
                concurrency,
            )
            before = len(seen)
            for br_lo, br_hi in get_price_seed_brackets():
                walk_hi = br_hi - 1 if br_hi is not None else None
                await self._walk_price_range(
                    rooms=rooms_key,
                    lo=br_lo,
                    hi=walk_hi,
                    seen=seen,
                    price_cap_per_bucket=price_cap_per_bucket,
                    max_pages_per_bucket=max_pages_per_bucket,
                    concurrency=concurrency,
                    on_bucket=on_bucket,
                    skip_buckets=skip_buckets,
                )
            room_collected = len(seen) - before
            logger.info(
                "yandex gate exhaustive: rooms=%s done -- collected %d (total unique=%d)",
                rooms_key,
                room_collected,
                len(seen),
            )
            if on_progress is not None:
                on_progress(len(seen))

        logger.info("yandex gate exhaustive: DONE -- total unique=%d lots", len(seen))
        return list(seen.values())

    async def _walk_price_range(
        self,
        *,
        rooms: str | None,
        lo: int,
        hi: int | None,
        seen: dict[str, ScrapedLot],
        price_cap_per_bucket: int,
        max_pages_per_bucket: int,
        concurrency: int = 4,
        on_bucket: Any = None,
        skip_buckets: set[str] | None = None,
        _depth: int = 0,
    ) -> None:
        """Recursive binary price-range partitioning.

        Probe page=1 -> read totalItems from pager.
        Closed bracket (hi is not None): totalItems > cap and bracket >= MIN_BRACKET
        -> split; else paginate leaf.
        Open bracket (hi is None): top seed-bracket without priceMax ceiling. Cannot
        split (no mid) -> paginate leaf directly (lux tail tiny, tail-loss accepted).
        Fallback: totalItems=None -> paginate-until-empty (safe degradation).
        skip_buckets: resume checkpoint -- skip leaf buckets already done.

        Реализация: единый движок `scraper_kit.pricing.walk_price_range`. Yandex-spec
        (probe gate-JSON page=1, degraded paginate-until-empty, leaf с first-wins
        дедупом и count-based on_bucket) — в callback'ах; пороги (cap, 500k min,
        depth<8) и политика DEGRADE — в BisectionConfig.
        """

        async def _probe(plo: int | None, phi: int | None, depth: int) -> ProbeResult:
            lo_param = plo if plo and plo > 0 else None
            hi_repr = "open" if phi is None else str(phi)
            probe_payload = await self._fetch_page_json(rooms, 1, lo_param, phi)
            await asyncio.sleep(self.request_delay_sec)

            if probe_payload is None:
                logger.warning(
                    "yandex gate: probe failed rooms=%s [%s, %s] depth=%d -- rotating + retry",
                    rooms,
                    plo,
                    hi_repr,
                    depth,
                )
                rotated = await self._rotate_ip()
                if rotated:
                    probe_payload = await self._fetch_page_json(rooms, 1, lo_param, phi)
                    await asyncio.sleep(self.request_delay_sec)

            total: int | None = None
            probe_lots: list[ScrapedLot] = []
            if probe_payload is not None:
                extracted = _extract_gate_data(probe_payload)
                if extracted is not None:
                    _, pager = extracted
                    total = pager.get("totalItems")
                    probe_lots = _parse_gate_json(probe_payload, page_param=1)
                else:
                    logger.warning(
                        "yandex gate: probe data extract failed rooms=%s [%s, %s]",
                        rooms,
                        plo,
                        hi_repr,
                    )
            return ProbeResult(count=total, payload=probe_lots)

        async def _degraded(plo: int | None, phi: int | None) -> None:
            # totalItems неизвестен → пагинируем до пустоты, не зная total.
            hi_repr = "open" if phi is None else str(phi)
            lo_param = plo if plo and plo > 0 else None
            logger.warning(
                "yandex gate: totalItems unknown rooms=%s [%s, %s] -- paginate until empty",
                rooms,
                plo,
                hi_repr,
            )
            bucket_key = _combo_label(rooms, plo, phi)
            if skip_buckets and bucket_key in skip_buckets:
                return
            page = 1
            pages_fetched = 0
            while pages_fetched < max_pages_per_bucket:
                payload = await self._fetch_page_json(rooms, page, lo_param, phi)
                await asyncio.sleep(self.request_delay_sec)
                if payload is None:
                    break
                lots = _parse_gate_json(payload, page_param=page)
                if not lots:
                    break
                for lot in lots:
                    if lot.source_id and lot.source_id not in seen:
                        seen[lot.source_id] = lot
                page += 1
                pages_fetched += 1
            if on_bucket is not None:
                on_bucket(bucket_key, len(seen))

        async def _leaf(plo: int | None, phi: int | None, result: ProbeResult) -> None:
            total = result.count
            assert total is not None  # DEGRADE-политика уводит None в _degraded
            lo_param = plo if plo and plo > 0 else None
            probe_lots: list[ScrapedLot] = result.payload or []
            bucket_key = _combo_label(rooms, plo, phi)
            if skip_buckets and bucket_key in skip_buckets:
                logger.debug("yandex gate: skip bucket %s (checkpoint)", bucket_key)
                return

            total_pages = min(
                math.ceil(total / 20),
                _GATE_MAX_PAGES_CAP,
                max_pages_per_bucket,
            )
            logger.info(
                "yandex gate: leaf bucket %s total=%d pages=%d", bucket_key, total, total_pages
            )

            # Add probe lots (page 1 already fetched)
            for lot in probe_lots:
                if lot.source_id and lot.source_id not in seen:
                    seen[lot.source_id] = lot

            if total_pages <= 1:
                if on_bucket is not None:
                    on_bucket(bucket_key, len(seen))
                return

            # Paginate pages 2..total_pages with concurrency
            sem = asyncio.Semaphore(concurrency)

            async def _fetch_leaf_page(pg: int) -> list[ScrapedLot]:
                async with sem:
                    payload = await self._fetch_page_json(rooms, pg, lo_param, phi)
                    await asyncio.sleep(self.request_delay_sec)
                    if payload is None:
                        return []
                    return _parse_gate_json(payload, page_param=pg)

            tasks = [_fetch_leaf_page(pg) for pg in range(2, total_pages + 1)]
            results = await asyncio.gather(*tasks)
            for page_lots in results:
                for lot in page_lots:
                    if lot.source_id and lot.source_id not in seen:
                        seen[lot.source_id] = lot

            if on_bucket is not None:
                on_bucket(bucket_key, len(seen))

        await walk_price_range(
            lo=lo,
            hi=hi,
            config=BisectionConfig(
                cap=price_cap_per_bucket,
                min_bracket=_YANDEX_SPLIT_MIN_BRACKET,
                max_depth=_YANDEX_MAX_SPLIT_DEPTH,
                probe_fail_policy=ProbeFailPolicy.DEGRADE,
                treat_zero_as_leaf=True,
            ),
            probe=_probe,
            on_leaf=_leaf,
            on_degraded=_degraded,
            depth=_depth,
        )


def _combo_label(rooms: str | None, lo: int | None, hi: int | None) -> str:
    """Уникальная метка бакета для checkpoint-логики."""
    return f"{rooms or 'any'}:{lo}-{hi}"
