"""Yandex.Nedvizhimost scraper (realty.yandex.ru) -- gate-API JSON parser.

Transport: system curl --compressed -x <socks5 proxy> (subprocess).
Proven to fetch gate-API payload in ~17s at 129 KB on prod (2026-06-17).

API path: GET https://realty.yandex.ru/gate/react-page/get/
  _pageType=search&_providers=react-search-results-data
  type=SELL&category=APARTMENT&newFlat=NO&rgid=559132 (EKB)
  roomsTotal=<int|STUDIO>&priceMin=...&priceMax=...&page=<1-based>

Response shape: response.search.offers.{entities[], pager{page,pageSize,totalItems,totalPages}}

Page param is 1-based: page=1 -> first page (pager.page=0 in response).
page=0 -> API returns error.  Paginate from page=1 to totalPages (inclusive).

Studio roomsTotal:
  Request: roomsTotal=STUDIO
  Response entities: roomsTotal=None -> mapped to rooms=0 (existing convention).
"""

from __future__ import annotations

import asyncio
import json
import logging
import math
from dataclasses import dataclass
from typing import Any
from urllib.parse import urlencode

from curl_cffi.requests import AsyncSession as _CurlCffiSession

from app.services.scraper_settings import get_scraper_delay
from app.services.scrapers.base import BaseScraper, ScrapedLot

logger = logging.getLogger(__name__)

DEFAULT_CITY = "ekaterinburg"
_GATE_MAX_PAGES_CAP = 50
_EKB_RGID = 559132
_GATE_URL = "https://realty.yandex.ru/gate/react-page/get/"
_CURL_UA = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"
)
_CURL_STATUS_MARKER = "\n###CURL_HTTP_STATUS:"

ROOM_GATE_PARAM: dict[str, str | int] = {
    "studio": "STUDIO",
    "1": 1,
    "2": 2,
    "3": 3,
    "4+": 4,
}

ROOM_PATH: dict[str, str] = {
    "studio": "studiya",
    "1": "odnokomnatnaya",
    "2": "dvuhkomnatnaya",
    "3": "trehkomnatnaya",
    "4+": "4-i-bolee",
}

DEFAULT_PRICE_RANGES: list[tuple[int | None, int | None]] = [
    (None, 5_000_000),
    (5_000_000, 7_000_000),
    (7_000_000, 10_000_000),
    (10_000_000, 15_000_000),
    (15_000_000, 25_000_000),
    (25_000_000, None),
]

_YANDEX_MAX_PRICE = 200_000_000
_YANDEX_MIN_BRACKET = 50_000
_GATE_PAGE_SIZE = 20
_YANDEX_TARPIT_MAX_RETRIES: int = 2


@dataclass
class _CurlResponse:
    status_code: int
    text: str


def _is_gate_error(payload: dict[str, Any]) -> bool:
    """Return True when gate-API returned an error object instead of search results."""
    return "error" in payload or "response" not in payload


def _extract_gate_data(
    payload: dict[str, Any],
) -> tuple[list[dict[str, Any]], dict[str, Any]] | None:
    """Extract (entities, pager) from valid gate-API response. None on missing path."""
    try:
        offers = payload["response"]["search"]["offers"]
        entities: list[dict[str, Any]] = offers["entities"]
        pager: dict[str, Any] = offers["pager"]
        return entities, pager
    except (KeyError, TypeError):
        return None


def _parse_gate_json(payload: dict[str, Any], page_param: int = 1) -> list[ScrapedLot]:
    """Parse gate-API JSON payload into a list of ScrapedLot.

    Verified field mapping (from .issdump/yandex_gate_api_sample.json + live prod 2026-06-17):
      offerId                         -> source_id
      url (//realty.yandex.ru/offer/) -> source_url (prepend "https:")
      price.value                     -> price_rub
      price.valuePerPart              -> price_per_m2
      area.value                      -> area_m2
      livingSpace.value               -> living_area_m2
      kitchenSpace.value              -> raw_payload["kitchen_area_m2"]
      roomsTotal (int | None)         -> rooms (None -> 0 for studio)
      floorsOffered[0]                -> floor
      floorsTotal                     -> total_floors
      building.builtYear              -> year_built
      building.buildingType           -> house_type (raw: "MONOLIT"/"BRICK"/etc.)
      ceilingHeight                   -> raw_payload["ceiling_height"]
      location.point.latitude/longitude -> lat / lon
      location.geocoderAddress        -> address (full, with house number)
      mainImages[]                    -> photo_urls (prepend "https:", up to 5)
      building.siteId                 -> house_ext_id (JK linkage)
      listing_segment = "vtorichka"   (all requests use newFlat=NO)
    """
    result = _extract_gate_data(payload)
    if result is None:
        logger.warning("yandex gate: response missing response.search.offers path")
        return []

    entities, _pager = result
    lots: list[ScrapedLot] = []
    for entity in entities:
        lot = _entity_to_lot(entity, page_param=page_param)
        if lot is not None:
            lots.append(lot)
    return lots


def _entity_to_lot(entity: dict[str, Any], page_param: int = 1) -> ScrapedLot | None:
    """Convert one gate-API entity dict to ScrapedLot. None on missing required fields."""
    try:
        offer_id = str(entity.get("offerId") or "")
        if not offer_id:
            return None

        url_raw = entity.get("url") or ""
        source_url = f"https:{url_raw}" if url_raw.startswith("//") else url_raw
        if not source_url:
            source_url = f"https://realty.yandex.ru/offer/{offer_id}/"

        price_dict = entity.get("price") or {}
        price_val = price_dict.get("value")
        if not price_val or price_val <= 0:
            return None
        price_rub = int(price_val)

        price_per_m2_raw = price_dict.get("valuePerPart")
        price_per_m2 = int(price_per_m2_raw) if price_per_m2_raw else None

        area_dict = entity.get("area") or {}
        area_m2_raw = area_dict.get("value")
        area_m2 = float(area_m2_raw) if area_m2_raw is not None else None

        living_dict = entity.get("livingSpace") or {}
        living_area_raw = living_dict.get("value")
        living_area_m2 = float(living_area_raw) if living_area_raw is not None else None

        kitchen_dict = entity.get("kitchenSpace") or {}
        kitchen_area_raw = kitchen_dict.get("value")

        rooms_raw = entity.get("roomsTotal")
        if rooms_raw is None:
            rooms = 0  # studio
        else:
            try:
                rooms = int(rooms_raw)
            except (TypeError, ValueError):
                rooms = None

        floors_offered = entity.get("floorsOffered") or []
        floor = int(floors_offered[0]) if floors_offered else None
        floors_total_raw = entity.get("floorsTotal")
        total_floors = int(floors_total_raw) if floors_total_raw is not None else None

        ceiling_height = entity.get("ceilingHeight")

        building = entity.get("building") or {}
        year_built_raw = building.get("builtYear")
        year_built = int(year_built_raw) if year_built_raw is not None else None
        house_type = building.get("buildingType")

        site_id = building.get("siteId")
        site_name = building.get("siteName")
        house_source: str | None = None
        house_ext_id: str | None = None
        if site_id:
            house_source = "yandex_realty_nb"
            house_ext_id = str(site_id)

        location = entity.get("location") or {}
        point = location.get("point") or {}
        lat_raw = point.get("latitude")
        lon_raw = point.get("longitude")
        lat = float(lat_raw) if lat_raw is not None else None
        lon = float(lon_raw) if lon_raw is not None else None

        address = location.get("geocoderAddress") or location.get("streetAddress") or None

        photo_urls: list[str] = []
        for img in entity.get("mainImages") or []:
            if img:
                photo_urls.append(f"https:{img}" if img.startswith("//") else img)
            if len(photo_urls) >= 5:
                break

        return ScrapedLot(
            source="yandex",
            source_url=source_url,
            source_id=offer_id,
            address=address,
            lat=lat,
            lon=lon,
            rooms=rooms,
            area_m2=area_m2,
            living_area_m2=living_area_m2,
            floor=floor,
            total_floors=total_floors,
            year_built=year_built,
            house_type=house_type,
            price_rub=price_rub,
            price_per_m2=price_per_m2,
            photo_urls=photo_urls,
            house_source=house_source,
            house_ext_id=house_ext_id,
            house_url=None,
            listing_segment="vtorichka",
            raw_payload={
                "page_param": page_param,
                "ceiling_height": ceiling_height,
                "kitchen_area_m2": kitchen_area_raw,
                "site_name": site_name,
                "offer_id": offer_id,
            },
        )
    except Exception:
        logger.exception("yandex gate: entity parse failed offer_id=%s", entity.get("offerId"))
        return None


class YandexRealtyScraper(BaseScraper):
    name = "yandex"
    base_url = "https://realty.yandex.ru"
    request_delay_sec = 5.0

    def __init__(self, city: str = DEFAULT_CITY) -> None:
        super().__init__()
        self.city = city
        self.request_delay_sec = get_scraper_delay(self.name)
        self._cffi_session: _CurlCffiSession | None = None
        self._cookies: dict[str, str] = {}

    async def __aenter__(self) -> YandexRealtyScraper:  # type: ignore[override]
        """Create curl_cffi session as fallback transport (dev / non-socks5 proxy).

        Primary transport: system curl subprocess over SOCKS5 (see _http_get).
        gate-API does not require cookies or heavy anti-bot headers.
        """
        from app.core.config import settings

        _proxy_url = settings.yandex_proxy_url
        _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
        self._cffi_session = _CurlCffiSession(
            impersonate="chrome120",
            cookies={},
            proxies=_proxies,
            headers={
                "Accept": "application/json",
                "X-Requested-With": "XMLHttpRequest",
                "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
                "Accept-Encoding": "gzip, deflate, br",
            },
        )
        return self

    async def __aexit__(self, *args: object) -> None:  # type: ignore[override]
        if self._cffi_session is not None:
            await self._cffi_session.close()
            self._cffi_session = None

    async def _curl_subprocess_get(self, url: str, proxy: str, timeout: int) -> _CurlResponse:
        """Fetch via system curl --compressed -x <socks5 proxy>.

        Appends HTTP status via -w marker and strips it from body.
        curl exit 28 (timeout) -> status_code=0.
        """
        args = [
            "curl",
            "-sS",
            "-L",
            "--compressed",
            "-x",
            proxy,
            "--max-time",
            str(timeout),
            "-A",
            _CURL_UA,
            "-H",
            "Accept: application/json",
            "-H",
            "X-Requested-With: XMLHttpRequest",
            "-H",
            "Accept-Language: ru-RU,ru;q=0.9,en;q=0.8",
            "-w",
            _CURL_STATUS_MARKER + "%{http_code}",
            url,
        ]
        try:
            proc = await asyncio.create_subprocess_exec(
                *args,
                stdout=asyncio.subprocess.PIPE,
                stderr=asyncio.subprocess.PIPE,
            )
            out, err = await asyncio.wait_for(proc.communicate(), timeout=timeout + 10)
        except TimeoutError:
            logger.warning("yandex curl: subprocess timeout url=%s", url)
            return _CurlResponse(status_code=0, text="")
        except Exception:
            logger.exception("yandex curl: subprocess failed url=%s", url)
            return _CurlResponse(status_code=0, text="")

        body = out.decode("utf-8", "replace")
        status = 0
        marker = _CURL_STATUS_MARKER
        idx = body.rfind(marker)
        if idx != -1:
            try:
                status = int(body[idx + len(marker) :].strip() or "0")
            except ValueError:
                status = 0
            body = body[:idx]

        if proc.returncode == 28:
            status = 0
        elif proc.returncode not in (0, None) and status == 0:
            logger.warning(
                "yandex curl: rc=%s url=%s err=%s",
                proc.returncode,
                url,
                err.decode("utf-8", "replace")[:200],
            )

        return _CurlResponse(status_code=status, text=body)

    async def _http_get(self, url: str, **kwargs: object) -> object:  # type: ignore[override]
        """Route GET to system curl (SOCKS5 proxy) or curl_cffi fallback."""
        from app.core.config import settings

        proxy = settings.yandex_proxy_url
        timeout = int(kwargs.get("timeout", 60) or 60)
        if proxy and proxy.startswith(("socks5://", "socks5h://")):
            return await self._curl_subprocess_get(url, proxy, timeout)

        if self._cffi_session is None:
            raise RuntimeError("YandexRealtyScraper must be used as async context manager")
        kwargs.setdefault("timeout", 60)
        return await self._cffi_session.get(url, **kwargs)

    async def _rotate_ip(self) -> bool:
        """Rotate mobile proxy IP via changeip URL. Returns True on success."""
        from app.core.config import settings as _settings

        rotate_url = _settings.yandex_proxy_rotate_url or _settings.avito_proxy_rotate_url
        if not rotate_url:
            return False
        sep = "&" if "?" in rotate_url else "?"
        try:
            async with _CurlCffiSession(timeout=30) as rot:
                await rot.get(f"{rotate_url}{sep}format=json")
            await asyncio.sleep(9)
            logger.info("yandex proxy: IP rotated via changeip")
            return True
        except Exception:
            logger.warning("yandex proxy: IP rotation failed", exc_info=True)
            return False

    def _build_url(
        self,
        page: int = 1,
        rooms: str | None = None,
        price_min: int | None = None,
        price_max: int | None = None,
    ) -> str:
        """Build gate-API URL. page: 1-based (page=0 -> API error)."""
        params: dict[str, str | int] = {
            "rgid": _EKB_RGID,
            "type": "SELL",
            "category": "APARTMENT",
            "newFlat": "NO",
            "_pageType": "search",
            "_providers": "react-search-results-data",
            "page": page,
        }
        if rooms and rooms in ROOM_GATE_PARAM:
            params["roomsTotal"] = ROOM_GATE_PARAM[rooms]
        if price_min is not None:
            params["priceMin"] = price_min
        if price_max is not None:
            params["priceMax"] = price_max
        return f"{_GATE_URL}?{urlencode(params)}"

    async def _fetch_page_json(
        self,
        rooms: str | None,
        page: int,
        price_min: int | None,
        price_max: int | None,
    ) -> dict[str, Any] | None:
        """Fetch one gate-API page. Returns parsed JSON dict or None on error/invalid."""
        url = self._build_url(page=page, rooms=rooms, price_min=price_min, price_max=price_max)
        try:
            resp = await self._http_get(url, timeout=60)
        except Exception:
            logger.exception("yandex gate: GET failed url=%s", url)
            return None
        if resp.status_code != 200:  # type: ignore[union-attr]
            logger.warning("yandex gate: HTTP %d url=%s", resp.status_code, url)  # type: ignore[union-attr]
            return None
        try:
            payload: dict[str, Any] = json.loads(resp.text)  # type: ignore[union-attr]
        except (json.JSONDecodeError, ValueError):
            logger.warning("yandex gate: JSON parse failed url=%s", url)
            return None
        if _is_gate_error(payload):
            logger.warning("yandex gate: error response url=%s keys=%s", url, list(payload.keys()))
            return None
        return payload

    async def fetch_around(
        self,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        page: int = 1,
        rooms: str | None = None,
        price_min: int | None = None,
        price_max: int | None = None,
    ) -> list[ScrapedLot]:
        """Fetch ONE page of gate-API results. lat/lon/radius_m ignored (uses rgid).

        page: 1-based. Tarpit resilience: status_code==0 or JSON error
        -> rotate IP + retry up to _YANDEX_TARPIT_MAX_RETRIES times.
        Non-tarpit failures (4xx/5xx) are not retried.
        """
        url = self._build_url(page=page, rooms=rooms, price_min=price_min, price_max=price_max)

        payload: dict[str, Any] | None = None
        for attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES):
            try:
                response = await self._http_get(url, timeout=60)
            except Exception:
                logger.exception("yandex gate fetch failed: %s", url)
                return []

            status = response.status_code  # type: ignore[union-attr]

            if status == 0:
                logger.warning(
                    "yandex gate: status=0 (tarpit?) rooms=%s page=%d"
                    " -- rotating IP + retry attempt %d",
                    rooms,
                    page,
                    attempt + 1,
                )
                await self._rotate_ip()
                await asyncio.sleep(2)
                continue

            if status != 200:
                logger.warning(
                    "yandex gate: HTTP %d rooms=%s page=%d url=%s", status, rooms, page, url
                )
                return []

            try:
                payload = json.loads(response.text)  # type: ignore[union-attr]
            except (json.JSONDecodeError, ValueError):
                logger.warning(
                    "yandex gate: JSON parse failed rooms=%s page=%d -- rotating + retry %d",
                    rooms,
                    page,
                    attempt + 1,
                )
                await self._rotate_ip()
                await asyncio.sleep(2)
                continue

            if _is_gate_error(payload):
                logger.warning(
                    "yandex gate: error payload rooms=%s page=%d keys=%s -- rotating + retry %d",
                    rooms,
                    page,
                    list(payload.keys()),
                    attempt + 1,
                )
                await self._rotate_ip()
                await asyncio.sleep(2)
                payload = None
                continue

            break

        if payload is None:
            logger.warning(
                "yandex gate: all retries exhausted rooms=%s page=%d -- returning empty",
                rooms,
                page,
            )
            return []

        lots = _parse_gate_json(payload, page_param=page)
        logger.info(
            "yandex gate page=%d rooms=%s price=[%s-%s]: %d cards",
            page,
            rooms or "all",
            price_min,
            price_max,
            len(lots),
        )
        await self.sleep_between_requests()
        return lots

    async def fetch_around_multi_room(
        self,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = _GATE_MAX_PAGES_CAP,
        rooms_list: list[str] | None = None,
        price_ranges: list[tuple[int | None, int | None]] | None = None,
        **_legacy_kwargs: Any,
    ) -> list[ScrapedLot]:
        """Fetch via rooms x price-range combos; paginate each combo to totalPages.

        Pagination driven by pager.totalPages from first page response.
        Each combo iterates page=1 .. min(totalPages, max_pages, _GATE_MAX_PAGES_CAP).
        Deduplicates by source_id/source_url across all combos.
        Legacy mode (rooms_list=None, price_ranges=None): single citywide sweep.
        """
        seen: dict[str, ScrapedLot] = {}

        if rooms_list is None and price_ranges is None:
            combos: list[tuple[str | None, int | None, int | None]] = [(None, None, None)]
        else:
            r_list = rooms_list or list(ROOM_PATH.keys())
            p_ranges = price_ranges or DEFAULT_PRICE_RANGES
            combos = [(r, lo, hi) for r in r_list for lo, hi in p_ranges]

        for rooms, price_min, price_max in combos:
            combo_label = _combo_label(rooms, price_min, price_max)
            total_pages: int | None = None

            for page in range(1, max_pages + 1):
                if page == 1:
                    payload_p1: dict[str, Any] | None = None
                    p1_url = self._build_url(
                        page=1, rooms=rooms, price_min=price_min, price_max=price_max
                    )
                    for _attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES):
                        try:
                            resp = await self._http_get(p1_url, timeout=60)
                        except Exception:
                            logger.exception("yandex gate combo fetch failed combo=%s", combo_label)
                            break
                        if resp.status_code == 0:  # type: ignore[union-attr]
                            logger.warning(
                                "yandex gate: tarpit combo=%s page=1 -- rotating", combo_label
                            )
                            await self._rotate_ip()
                            await asyncio.sleep(2)
                            continue
                        if resp.status_code != 200:  # type: ignore[union-attr]
                            logger.warning(
                                "yandex gate: HTTP %d combo=%s page=1",
                                resp.status_code,
                                combo_label,  # type: ignore[union-attr]
                            )
                            break
                        try:
                            payload_p1 = json.loads(resp.text)  # type: ignore[union-attr]
                        except (json.JSONDecodeError, ValueError):
                            logger.warning(
                                "yandex gate: JSON parse error combo=%s page=1", combo_label
                            )
                            payload_p1 = None
                            await self._rotate_ip()
                            await asyncio.sleep(2)
                            continue
                        if _is_gate_error(payload_p1):
                            logger.warning(
                                "yandex gate: error payload combo=%s page=1", combo_label
                            )
                            payload_p1 = None
                            break
                        break

                    if payload_p1 is None:
                        logger.debug(
                            "yandex gate combo [%s] page=1: failed -- skipping", combo_label
                        )
                        break

                    result = _extract_gate_data(payload_p1)
                    if result is None:
                        break
                    _entities_p1, pager_p1 = result
                    total_pages = min(
                        pager_p1.get("totalPages", 1),
                        max_pages,
                        _GATE_MAX_PAGES_CAP,
                    )
                    lots_p1 = _parse_gate_json(payload_p1, page_param=1)
                    if not lots_p1:
                        logger.debug(
                            "yandex gate combo [%s] page=1: empty -- stopping", combo_label
                        )
                        break
                    for lot in lots_p1:
                        key = lot.source_id or lot.source_url
                        if key and key not in seen:
                            seen[key] = lot
                    await self.sleep_between_requests()
                    if total_pages <= 1:
                        break
                    continue

                if total_pages is not None and page > total_pages:
                    break

                lots = await self.fetch_around(
                    lat,
                    lon,
                    radius_m,
                    page=page,
                    rooms=rooms,
                    price_min=price_min,
                    price_max=price_max,
                )
                if not lots:
                    logger.debug(
                        "yandex gate combo [%s] page=%d: empty -- stopping",
                        combo_label,
                        page,
                    )
                    break
                for lot in lots:
                    key = lot.source_id or lot.source_url
                    if key and key not in seen:
                        seen[key] = lot

        logger.info(
            "yandex gate aggregate: %d unique lots (%d combos)",
            len(seen),
            len(combos),
        )
        return list(seen.values())

    async def fetch_all_secondary(
        self,
        *,
        rooms_buckets: list[str] | None = None,
        price_cap_per_bucket: int = 500,
        max_pages_per_bucket: int = _GATE_MAX_PAGES_CAP,
        concurrency: int = 4,
        on_bucket: Any = None,
        on_progress: Any = None,
        skip_buckets: set[str] | None = None,
    ) -> list[ScrapedLot]:
        """Exhaustive EKB vtorichka load via KOMNATNOST x TSENA partitioning.

        Uses gate-API JSON (totalItems from pager) instead of DOM state extraction.
        """
        _buckets = rooms_buckets if rooms_buckets is not None else list(ROOM_PATH.keys())
        seen: dict[str, ScrapedLot] = {}

        for rooms_key in _buckets:
            logger.info(
                "yandex gate exhaustive: starting rooms=%s (price_cap=%d concurrency=%d)",
                rooms_key,
                price_cap_per_bucket,
                concurrency,
            )
            before = len(seen)
            await self._walk_price_range(
                rooms=rooms_key,
                lo=0,
                hi=_YANDEX_MAX_PRICE,
                seen=seen,
                price_cap_per_bucket=price_cap_per_bucket,
                max_pages_per_bucket=max_pages_per_bucket,
                concurrency=concurrency,
                on_bucket=on_bucket,
                skip_buckets=skip_buckets,
            )
            room_collected = len(seen) - before
            logger.info(
                "yandex gate exhaustive: rooms=%s done -- collected %d (total unique=%d)",
                rooms_key,
                room_collected,
                len(seen),
            )
            if on_progress is not None:
                on_progress(len(seen))

        logger.info("yandex gate exhaustive: DONE -- total unique=%d lots", len(seen))
        return list(seen.values())

    async def _walk_price_range(
        self,
        *,
        rooms: str | None,
        lo: int,
        hi: int,
        seen: dict[str, ScrapedLot],
        price_cap_per_bucket: int,
        max_pages_per_bucket: int,
        concurrency: int = 4,
        on_bucket: Any = None,
        skip_buckets: set[str] | None = None,
        _depth: int = 0,
    ) -> None:
        """Recursive binary price-range partitioning.

        Probe page=1 -> read totalItems from pager.
        totalItems > cap and bracket >= MIN_BRACKET -> split; else paginate leaf.
        Fallback: totalItems=None -> paginate-until-empty (safe degradation).
        skip_buckets: resume checkpoint -- skip leaf buckets already done.
        """
        _lo_param = lo if lo > 0 else None

        probe_payload = await self._fetch_page_json(rooms, 1, _lo_param, hi)
        await asyncio.sleep(self.request_delay_sec)

        total: int | None = None
        probe_lots: list[ScrapedLot] = []

        if probe_payload is None:
            logger.warning(
                "yandex gate: probe failed rooms=%s [%d, %d] depth=%d -- rotating + retry",
                rooms,
                lo,
                hi,
                _depth,
            )
            rotated = await self._rotate_ip()
            if rotated:
                probe_payload = await self._fetch_page_json(rooms, 1, _lo_param, hi)
                await asyncio.sleep(self.request_delay_sec)

        if probe_payload is not None:
            result = _extract_gate_data(probe_payload)
            if result is not None:
                _, pager = result
                total = pager.get("totalItems")
                probe_lots = _parse_gate_json(probe_payload, page_param=1)
            else:
                logger.warning(
                    "yandex gate: probe data extract failed rooms=%s [%d, %d]", rooms, lo, hi
                )

        if total is None:
            # Degraded path: paginate-until-empty without knowing total
            logger.warning(
                "yandex gate: totalItems unknown rooms=%s [%d, %d] -- paginate until empty",
                rooms,
                lo,
                hi,
            )
            page = 1
            bucket_key = _combo_label(rooms, lo, hi)
            if skip_buckets and bucket_key in skip_buckets:
                return
            pages_fetched = 0
            while pages_fetched < max_pages_per_bucket:
                payload = await self._fetch_page_json(rooms, page, _lo_param, hi)
                await asyncio.sleep(self.request_delay_sec)
                if payload is None:
                    break
                lots = _parse_gate_json(payload, page_param=page)
                if not lots:
                    break
                for lot in lots:
                    if lot.source_id and lot.source_id not in seen:
                        seen[lot.source_id] = lot
                page += 1
                pages_fetched += 1
            if on_bucket is not None:
                on_bucket(bucket_key, len(seen))
            return

        # Split or paginate
        cap = price_cap_per_bucket
        _min_bracket = 500_000
        bracket = hi - lo

        if total > cap and bracket >= _min_bracket and _depth < 8:
            mid = (lo + hi) // 2
            await self._walk_price_range(
                rooms=rooms,
                lo=lo,
                hi=mid,
                seen=seen,
                price_cap_per_bucket=cap,
                max_pages_per_bucket=max_pages_per_bucket,
                concurrency=concurrency,
                on_bucket=on_bucket,
                skip_buckets=skip_buckets,
                _depth=_depth + 1,
            )
            await self._walk_price_range(
                rooms=rooms,
                lo=mid + 1,
                hi=hi,
                seen=seen,
                price_cap_per_bucket=cap,
                max_pages_per_bucket=max_pages_per_bucket,
                concurrency=concurrency,
                on_bucket=on_bucket,
                skip_buckets=skip_buckets,
                _depth=_depth + 1,
            )
            return

        # Leaf bucket: paginate
        bucket_key = _combo_label(rooms, lo, hi)
        if skip_buckets and bucket_key in skip_buckets:
            logger.debug("yandex gate: skip bucket %s (checkpoint)", bucket_key)
            return

        total_pages = min(
            math.ceil(total / 20),
            _GATE_MAX_PAGES_CAP,
            max_pages_per_bucket,
        )
        logger.info("yandex gate: leaf bucket %s total=%d pages=%d", bucket_key, total, total_pages)

        # Add probe lots (page 1 already fetched)
        for lot in probe_lots:
            if lot.source_id and lot.source_id not in seen:
                seen[lot.source_id] = lot

        if total_pages <= 1:
            if on_bucket is not None:
                on_bucket(bucket_key, len(seen))
            return

        # Paginate pages 2..total_pages with concurrency
        sem = asyncio.Semaphore(concurrency)

        async def _fetch_leaf_page(pg: int) -> list[ScrapedLot]:
            async with sem:
                payload = await self._fetch_page_json(rooms, pg, _lo_param, hi)
                await asyncio.sleep(self.request_delay_sec)
                if payload is None:
                    return []
                return _parse_gate_json(payload, page_param=pg)

        tasks = [_fetch_leaf_page(pg) for pg in range(2, total_pages + 1)]
        results = await asyncio.gather(*tasks)
        for page_lots in results:
            for lot in page_lots:
                if lot.source_id and lot.source_id not in seen:
                    seen[lot.source_id] = lot

        if on_bucket is not None:
            on_bucket(bucket_key, len(seen))


def _combo_label(rooms: str | None, lo: int | None, hi: int | None) -> str:
    """Уникальная метка бакета для checkpoint-логики."""
    return f"{rooms or 'any'}:{lo}-{hi}"