"""Unit tests for YandexRealtyScraper gate-API JSON parser.

Replaced DOM/HTML SERP tests (2026-06-17): new transport uses
GET realty.yandex.ru/gate/react-page/get/ (JSON gate-API) instead of HTML SERP.
"""

import json
from unittest.mock import AsyncMock, MagicMock, patch

import pytest

from app.services.scrapers.base import ScrapedLot
from app.services.scrapers.yandex_realty import (
    _CURL_STATUS_MARKER,
    _GATE_MAX_PAGES_CAP,
    DEFAULT_CITY,
    DEFAULT_PRICE_RANGES,
    ROOM_PATH,
    YandexRealtyScraper,
    _combo_label,
    _CurlResponse,
    _entity_to_lot,
    _extract_gate_data,
    _is_gate_error,
    _parse_gate_json,
)

# ---------------------------------------------------------------------------
# Minimal gate-API JSON fixtures (derived from real captured sample 2026-06-17)
# pager shape: {page, pageSize, totalItems, totalPages}
# ---------------------------------------------------------------------------

_ENTITY_FULL: dict = {
    "offerId": 4740475460451078271,
    "url": "//realty.yandex.ru/offer/4740475460451078271",
    "price": {
        "currency": "RUR",
        "value": 4500000,
        "valuePerPart": 119048,
    },
    "area": {"value": 37.8, "unit": "SQUARE_METER"},
    "livingSpace": {"value": 20, "unit": "SQUARE_METER"},
    "kitchenSpace": {"value": 12.7, "unit": "SQUARE_METER"},
    "roomsTotal": 1,
    "floorsOffered": [8],
    "floorsTotal": 25,
    "ceilingHeight": 2.7,
    "building": {
        "builtYear": 2016,
        "buildingType": "MONOLIT",
        "siteId": 280938,
        "siteName": "Peremen",
    },
    "location": {
        "geocoderAddress": "Ekaterinburg, ulitsa Evgenia Savkova, 8",
        "point": {"latitude": 56.79512, "longitude": 60.49186},
    },
    "mainImages": [
        "//avatars.mds.yandex.net/get-realty-offers/11904018/abc/main",
        "//avatars.mds.yandex.net/get-realty-offers/14717586/def/main",
    ],
}

_ENTITY_STUDIO: dict = {
    "offerId": 9999999,
    "url": "//realty.yandex.ru/offer/9999999",
    "price": {"value": 3200000, "valuePerPart": 114285},
    "area": {"value": 28.0},
    "livingSpace": None,
    "kitchenSpace": None,
    "roomsTotal": None,  # studio -> rooms=0
    "floorsOffered": [2],
    "floorsTotal": 9,
    "ceilingHeight": None,
    "building": {"builtYear": 2010, "buildingType": "PANEL", "siteId": None, "siteName": None},
    "location": {
        "geocoderAddress": "Ekaterinburg, ulitsa Lenina, 5",
        "point": {"latitude": 56.838, "longitude": 60.597},
    },
    "mainImages": [],
}

_ENTITY_NO_PRICE: dict = {
    "offerId": 1111,
    "url": "//realty.yandex.ru/offer/1111",
    "price": {"value": 0},
    "area": {"value": 40.0},
    "roomsTotal": 1,
    "floorsOffered": [3],
    "floorsTotal": 5,
    "building": {},
    "location": {"point": {"latitude": 56.8, "longitude": 60.6}},
    "mainImages": [],
}

_ENTITY_NO_OFFER_ID: dict = {
    "offerId": None,
    "price": {"value": 5000000},
    "area": {"value": 50.0},
}


def _make_gate_payload(entities: list, pager: dict | None = None) -> dict:
    """Wrap entities + pager into gate-API response shape."""
    if pager is None:
        pager = {"page": 0, "pageSize": 20, "totalItems": len(entities), "totalPages": 1}
    return {
        "response": {
            "search": {
                "offers": {
                    "entities": entities,
                    "pager": pager,
                }
            }
        }
    }


# ---------------------------------------------------------------------------
# PART A: pure helper functions
# ---------------------------------------------------------------------------


def test_is_gate_error_on_error_key():
    assert _is_gate_error({"error": "bad request"}) is True


def test_is_gate_error_on_missing_response():
    assert _is_gate_error({"something": "else"}) is True


def test_is_gate_error_on_valid_payload():
    payload = _make_gate_payload([_ENTITY_FULL])
    assert _is_gate_error(payload) is False


def test_extract_gate_data_returns_entities_and_pager():
    payload = _make_gate_payload([_ENTITY_FULL, _ENTITY_STUDIO])
    result = _extract_gate_data(payload)
    assert result is not None
    entities, pager = result
    assert len(entities) == 2
    assert "totalItems" in pager


def test_extract_gate_data_returns_none_on_bad_shape():
    assert _extract_gate_data({}) is None
    assert _extract_gate_data({"response": {}}) is None
    assert _extract_gate_data({"response": {"search": {}}}) is None


# ---------------------------------------------------------------------------
# PART B: _entity_to_lot field mapping
# ---------------------------------------------------------------------------


def test_entity_to_lot_full_entity():
    lot = _entity_to_lot(_ENTITY_FULL)
    assert lot is not None
    assert lot.source == "yandex"
    assert lot.source_id == "4740475460451078271"
    assert lot.source_url == "https://realty.yandex.ru/offer/4740475460451078271"
    assert lot.price_rub == 4_500_000
    assert lot.price_per_m2 == 119_048
    assert lot.area_m2 == pytest.approx(37.8)
    assert lot.living_area_m2 == pytest.approx(20.0)
    assert lot.rooms == 1
    assert lot.floor == 8
    assert lot.total_floors == 25
    assert lot.year_built == 2016
    assert lot.house_type == "MONOLIT"
    assert lot.lat == pytest.approx(56.79512)
    assert lot.lon == pytest.approx(60.49186)
    assert lot.address == "Ekaterinburg, ulitsa Evgenia Savkova, 8"
    assert lot.house_source == "yandex_realty_nb"
    assert lot.house_ext_id == "280938"
    assert lot.listing_segment == "vtorichka"
    assert len(lot.photo_urls) == 2
    assert all(u.startswith("https://avatars.mds.yandex.net") for u in lot.photo_urls)
    assert lot.raw_payload is not None
    assert lot.raw_payload["ceiling_height"] == pytest.approx(2.7)
    assert lot.raw_payload["kitchen_area_m2"] == pytest.approx(12.7)


def test_entity_to_lot_studio_rooms_zero():
    lot = _entity_to_lot(_ENTITY_STUDIO)
    assert lot is not None
    assert lot.rooms == 0
    assert lot.area_m2 == pytest.approx(28.0)
    assert lot.floor == 2
    assert lot.total_floors == 9
    assert lot.house_source is None
    assert lot.house_ext_id is None
    assert lot.photo_urls == []


def test_entity_to_lot_no_price_returns_none():
    lot = _entity_to_lot(_ENTITY_NO_PRICE)
    assert lot is None


def test_entity_to_lot_no_offer_id_returns_none():
    lot = _entity_to_lot(_ENTITY_NO_OFFER_ID)
    assert lot is None


def test_entity_to_lot_photo_url_prefix():
    """mainImages starting with // must get https: prepended."""
    lot = _entity_to_lot(_ENTITY_FULL)
    assert lot is not None
    # Both URLs must start with https:// (not protocol-relative //)
    assert all(u.startswith("https://") for u in lot.photo_urls)
    assert not any(u.startswith("//") for u in lot.photo_urls)


def test_entity_to_lot_photo_capped_at_5():
    entity = dict(_ENTITY_FULL)
    entity["mainImages"] = [f"//avatars.mds.yandex.net/img{i}/main" for i in range(10)]
    lot = _entity_to_lot(entity)
    assert lot is not None
    assert len(lot.photo_urls) == 5


def test_entity_to_lot_url_fallback_when_missing():
    """When url field absent, fall back to constructed /offer/<id>/ URL."""
    entity = dict(_ENTITY_FULL)
    entity["url"] = None
    lot = _entity_to_lot(entity)
    assert lot is not None
    expected_id = _ENTITY_FULL["offerId"]
    assert lot.source_url == f"https://realty.yandex.ru/offer/{expected_id}/"


def test_entity_to_lot_no_house_when_site_id_absent():
    """Entity without siteId produces house_source=None, house_ext_id=None."""
    entity = dict(_ENTITY_FULL)
    entity["building"] = {"builtYear": 2015, "buildingType": "BRICK", "siteId": None}
    lot = _entity_to_lot(entity)
    assert lot is not None
    assert lot.house_source is None
    assert lot.house_ext_id is None


def test_entity_to_lot_page_param_in_raw():
    """page_param passed to _entity_to_lot appears in raw_payload."""
    lot = _entity_to_lot(_ENTITY_FULL, page_param=3)
    assert lot is not None
    assert lot.raw_payload["page_param"] == 3


# ---------------------------------------------------------------------------
# PART C: _parse_gate_json (list-level)
# ---------------------------------------------------------------------------


def test_parse_gate_json_returns_list():
    payload = _make_gate_payload([_ENTITY_FULL, _ENTITY_STUDIO])
    lots = _parse_gate_json(payload)
    assert len(lots) == 2
    assert all(isinstance(lot, ScrapedLot) for lot in lots)


def test_parse_gate_json_skips_invalid():
    """Entities that fail validation (no price, no id) are silently skipped."""
    payload = _make_gate_payload([_ENTITY_FULL, _ENTITY_NO_PRICE, _ENTITY_NO_OFFER_ID])
    lots = _parse_gate_json(payload)
    assert len(lots) == 1
    assert lots[0].source_id == str(_ENTITY_FULL["offerId"])


def test_parse_gate_json_empty_entities():
    payload = _make_gate_payload([])
    lots = _parse_gate_json(payload)
    assert lots == []


def test_parse_gate_json_bad_payload():
    """Error/missing-path payload returns empty list without raising."""
    assert _parse_gate_json({}) == []
    assert _parse_gate_json({"error": "bad"}) == []


# ---------------------------------------------------------------------------
# PART D: _build_url gate-API URL shape
# ---------------------------------------------------------------------------


def test_build_url_contains_required_params():
    s = YandexRealtyScraper()
    url = s._build_url(page=1)
    assert "realty.yandex.ru/gate/react-page/get/" in url
    assert "rgid=559132" in url
    assert "type=SELL" in url
    assert "category=APARTMENT" in url
    assert "newFlat=NO" in url
    assert "_pageType=search" in url
    assert "_providers=react-search-results-data" in url
    assert "page=1" in url


def test_build_url_page_1_based():
    """page param is 1-based; page=1 is first page."""
    s = YandexRealtyScraper()
    url1 = s._build_url(page=1)
    url2 = s._build_url(page=2)
    assert "page=1" in url1
    assert "page=2" in url2


def test_build_url_rooms_studio():
    s = YandexRealtyScraper()
    url = s._build_url(page=1, rooms="studio")
    assert "roomsTotal=STUDIO" in url


def test_build_url_rooms_1():
    s = YandexRealtyScraper()
    url = s._build_url(page=1, rooms="1")
    assert "roomsTotal=1" in url


def test_build_url_rooms_4plus():
    s = YandexRealtyScraper()
    url = s._build_url(page=1, rooms="4+")
    assert "roomsTotal=4" in url


def test_build_url_price_range():
    s = YandexRealtyScraper()
    url = s._build_url(page=1, price_min=5_000_000, price_max=7_000_000)
    assert "priceMin=5000000" in url
    assert "priceMax=7000000" in url


def test_build_url_unknown_rooms_no_rooms_param():
    """Unknown rooms key must not produce a roomsTotal param."""
    s = YandexRealtyScraper()
    url = s._build_url(page=1, rooms="99-room-penthouse")
    assert "roomsTotal" not in url


def test_build_url_combos_are_unique():
    """All rooms x price-range combos produce distinct URLs."""
    s = YandexRealtyScraper()
    urls = set()
    for rooms in ROOM_PATH.keys():
        for lo, hi in DEFAULT_PRICE_RANGES:
            urls.add(s._build_url(page=1, rooms=rooms, price_min=lo, price_max=hi))
    expected = len(ROOM_PATH) * len(DEFAULT_PRICE_RANGES)
    assert len(urls) == expected, f"URL collisions: {expected} combos but {len(urls)} unique"


def test_default_city():
    assert DEFAULT_CITY == "ekaterinburg"


def test_gate_max_pages_cap():
    assert _GATE_MAX_PAGES_CAP == 50


# ---------------------------------------------------------------------------
# PART E: _combo_label
# ---------------------------------------------------------------------------


def test_combo_label_none_none():
    result = _combo_label(None, 0, 200_000_000)
    assert isinstance(result, str)
    assert "any" in result


def test_combo_label_room_and_price():
    result = _combo_label("1", 5_000_000, 7_000_000)
    assert "1" in result
    assert "5000000" in result
    assert "7000000" in result


def test_combo_label_different_ranges_differ():
    k1 = _combo_label("1", 5_000_000, 7_000_000)
    k2 = _combo_label("1", 7_000_000, 10_000_000)
    k3 = _combo_label("2", 5_000_000, 7_000_000)
    assert k1 != k2
    assert k1 != k3


# ---------------------------------------------------------------------------
# PART F: transport layer
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_aexit_closes_session():
    s = YandexRealtyScraper()
    mock_session = AsyncMock()
    s._cffi_session = mock_session
    await s.__aexit__(None, None, None)
    mock_session.close.assert_awaited_once()
    assert s._cffi_session is None


@pytest.mark.asyncio
async def test_http_get_raises_when_no_context():
    """_http_get must raise RuntimeError outside async context manager (cffi path)."""
    s = YandexRealtyScraper()
    mock_settings = MagicMock()
    mock_settings.yandex_proxy_url = None
    with patch.dict("sys.modules", {"app.core.config": MagicMock(settings=mock_settings)}):
        with pytest.raises(RuntimeError, match="async context manager"):
            await s._http_get("https://realty.yandex.ru/test/")


@pytest.mark.asyncio
async def test_http_get_routes_socks5_to_curl_subprocess():
    s = YandexRealtyScraper()
    sentinel = _CurlResponse(status_code=200, text="{}")

    mock_settings = MagicMock()
    mock_settings.yandex_proxy_url = "socks5h://x:y@h:1"

    cffi_mock = AsyncMock()
    s._cffi_session = cffi_mock

    async def fake_curl_subprocess(url: str, proxy: str, timeout: int) -> object:
        return sentinel

    with patch.dict("sys.modules", {"app.core.config": MagicMock(settings=mock_settings)}):
        with patch.object(s, "_curl_subprocess_get", side_effect=fake_curl_subprocess) as mock_curl:
            result = await s._http_get("https://realty.yandex.ru/test/", timeout=30)

    mock_curl.assert_awaited_once()
    assert result is sentinel
    cffi_mock.get.assert_not_called()


@pytest.mark.asyncio
async def test_curl_subprocess_get_parses_status_and_strips_marker():
    s = YandexRealtyScraper()
    fake_body = b"{" + b'"response":{}}'
    fake_stdout = fake_body + _CURL_STATUS_MARKER.encode() + b"200"

    fake_proc = MagicMock()
    fake_proc.returncode = 0
    fake_proc.communicate = AsyncMock(return_value=(fake_stdout, b""))

    with patch("asyncio.create_subprocess_exec", return_value=fake_proc):
        resp = await s._curl_subprocess_get(
            "https://realty.yandex.ru/test/", "socks5h://x:y@h:1", 30
        )

    assert isinstance(resp, _CurlResponse)
    assert resp.status_code == 200
    assert resp.text == "{" + '"response":{}}'


@pytest.mark.asyncio
async def test_http_get_http_proxy_uses_cffi_not_curl():
    """When proxy is http://, must NOT call _curl_subprocess_get."""
    s = YandexRealtyScraper()
    sentinel_resp = MagicMock()
    sentinel_resp.status_code = 200
    sentinel_resp.text = "{}"

    cffi_mock = AsyncMock()
    cffi_mock.get = AsyncMock(return_value=sentinel_resp)
    s._cffi_session = cffi_mock

    mock_settings = MagicMock()
    mock_settings.yandex_proxy_url = "http://proxy.example.com:8080"

    with patch.dict("sys.modules", {"app.core.config": MagicMock(settings=mock_settings)}):
        with patch.object(s, "_curl_subprocess_get") as mock_curl:
            result = await s._http_get("https://realty.yandex.ru/test/", timeout=30)

    mock_curl.assert_not_called()
    cffi_mock.get.assert_awaited_once()
    assert result is sentinel_resp


# ---------------------------------------------------------------------------
# PART G: fetch_around — tarpit resilience
# ---------------------------------------------------------------------------


def _valid_gate_text(entity: dict | None = None) -> str:
    ent = entity or _ENTITY_FULL
    pager = {"page": 0, "pageSize": 20, "totalItems": 1, "totalPages": 1}
    return json.dumps(_make_gate_payload([ent], pager))


@pytest.mark.asyncio
async def test_fetch_around_returns_lots_on_200_json():
    s = YandexRealtyScraper()
    body = _valid_gate_text()

    async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
        return _CurlResponse(status_code=200, text=body)

    with patch.object(s, "_http_get", side_effect=fake_http_get):
        with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
            lots = await s.fetch_around(lat=56.8, lon=60.6, rooms="1")

    assert len(lots) == 1
    assert lots[0].source_id == str(_ENTITY_FULL["offerId"])


@pytest.mark.asyncio
async def test_fetch_around_rotates_and_retries_on_status_0():
    """fetch_around rotates IP and retries on status_code==0 (tarpit)."""
    s = YandexRealtyScraper()
    call_count = 0
    body = _valid_gate_text()

    async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
        nonlocal call_count
        call_count += 1
        if call_count == 1:
            return _CurlResponse(status_code=0, text="")
        return _CurlResponse(status_code=200, text=body)

    with patch.object(s, "_http_get", side_effect=fake_http_get):
        with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate:
            with patch("asyncio.sleep", new_callable=AsyncMock):
                with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
                    lots = await s.fetch_around(lat=56.84, lon=60.60, rooms="1")

    assert mock_rotate.call_count == 1
    assert call_count == 2
    assert len(lots) >= 1


@pytest.mark.asyncio
async def test_fetch_around_rotates_on_json_error():
    """fetch_around rotates when response body is not valid JSON."""
    s = YandexRealtyScraper()
    call_count = 0
    body = _valid_gate_text()

    async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
        nonlocal call_count
        call_count += 1
        if call_count == 1:
            return _CurlResponse(status_code=200, text="<html>not json</html>")
        return _CurlResponse(status_code=200, text=body)

    with patch.object(s, "_http_get", side_effect=fake_http_get):
        with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate:
            with patch("asyncio.sleep", new_callable=AsyncMock):
                with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
                    lots = await s.fetch_around(lat=56.84, lon=60.60, rooms="2")

    assert mock_rotate.call_count == 1
    assert len(lots) >= 1


@pytest.mark.asyncio
async def test_fetch_around_exhausts_retries_returns_empty():
    """fetch_around returns [] when all retries exhausted."""
    s = YandexRealtyScraper()

    async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
        return _CurlResponse(status_code=0, text="")

    with patch.object(s, "_http_get", side_effect=fake_http_get):
        with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate:
            with patch("asyncio.sleep", new_callable=AsyncMock):
                lots = await s.fetch_around(lat=56.84, lon=60.60, rooms="1")

    assert mock_rotate.call_count >= 1
    assert lots == []


@pytest.mark.asyncio
async def test_fetch_around_no_rotate_on_404():
    """fetch_around does NOT rotate on 404 -- only on tarpit (status==0) / bad JSON."""
    s = YandexRealtyScraper()

    async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
        return _CurlResponse(status_code=404, text="Not Found")

    with patch.object(s, "_http_get", side_effect=fake_http_get):
        with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate:
            lots = await s.fetch_around(lat=56.84, lon=60.60)

    assert mock_rotate.call_count == 0
    assert lots == []


# ---------------------------------------------------------------------------
# PART H: fetch_around_multi_room
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_fetch_around_multi_room_dedup():
    """Multi-room fetch deduplicates offers seen in multiple combos.

    fetch_around_multi_room fetches page=1 inline via _http_get, not via
    fetch_around. We mock _http_get to return JSON payloads with shared/unique
    offer IDs to verify deduplication.
    """
    s = YandexRealtyScraper()

    # Combo 0: shared_offer + unique_0
    # Combo 1: shared_offer + unique_1
    entity_shared = dict(
        _ENTITY_FULL, offerId="shared_offer_1234", url="//realty.yandex.ru/offer/shared_offer_1234"
    )
    entity_unique_0 = dict(
        _ENTITY_FULL, offerId="unique_offer_0", url="//realty.yandex.ru/offer/unique_offer_0"
    )
    entity_unique_1 = dict(
        _ENTITY_FULL, offerId="unique_offer_1", url="//realty.yandex.ru/offer/unique_offer_1"
    )

    pager = {"page": 0, "pageSize": 20, "totalItems": 2, "totalPages": 1}
    payload_combo0 = _make_gate_payload([entity_shared, entity_unique_0], pager)
    payload_combo1 = _make_gate_payload([entity_shared, entity_unique_1], pager)

    combo_payloads = [payload_combo0, payload_combo1]
    call_idx = 0

    async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
        nonlocal call_idx
        idx = call_idx
        call_idx += 1
        if idx < len(combo_payloads):
            return _CurlResponse(status_code=200, text=json.dumps(combo_payloads[idx]))
        return _CurlResponse(status_code=200, text=json.dumps(_make_gate_payload([])))

    mock_settings = MagicMock()
    mock_settings.yandex_proxy_url = None

    with patch.dict("sys.modules", {"app.core.config": MagicMock(settings=mock_settings)}):
        with patch.object(s, "_http_get", side_effect=fake_http_get):
            with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
                lots = await s.fetch_around_multi_room(
                    lat=0.0,
                    lon=0.0,
                    rooms_list=["1"],
                    price_ranges=[(None, 5_000_000), (5_000_000, 7_000_000)],
                    max_pages=1,
                )

    ids = {lot.source_id for lot in lots}
    assert "shared_offer_1234" in ids
    assert len(ids) == 3


@pytest.mark.asyncio
async def test_fetch_around_multi_room_legacy_single_sweep():
    """Legacy mode (no rooms_list/price_ranges) uses single citywide sweep."""
    s = YandexRealtyScraper()
    calls: list[dict] = []

    async def fake_fetch_around(
        lat, lon, radius_m=1000, page=1, rooms=None, price_min=None, price_max=None
    ):
        calls.append({"page": page, "rooms": rooms, "price_min": price_min})
        return []

    with patch.object(s, "fetch_around", side_effect=fake_fetch_around):
        await s.fetch_around_multi_room(lat=0.0, lon=0.0, max_pages=3)

    assert all(c["rooms"] is None for c in calls)
    assert all(c["price_min"] is None for c in calls)


# ---------------------------------------------------------------------------
# PART I: exhaustive load -- fetch_all_secondary + _walk_price_range
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_walk_price_range_splits_when_total_exceeds_cap():
    """When totalItems > cap, _walk_price_range splits into sub-ranges recursively."""
    s = YandexRealtyScraper()

    call_ranges: list[tuple] = []
    call_count = 0

    async def fake_fetch_page_json(rooms, page, price_min, price_max):
        nonlocal call_count
        call_count += 1
        call_ranges.append((price_min, price_max))
        # First call (wide range): 600 items -> triggers split
        total = 600 if call_count == 1 else 10
        entities = [dict(_ENTITY_FULL, offerId=f"offer_{call_count}")]
        pager = {
            "page": 0,
            "pageSize": 20,
            "totalItems": total,
            "totalPages": max(1, total // 20),
        }
        return _make_gate_payload(entities, pager)

    seen: dict = {}
    with patch.object(s, "_fetch_page_json", side_effect=fake_fetch_page_json):
        with patch("asyncio.sleep", new_callable=AsyncMock):
            await s._walk_price_range(
                rooms="1",
                lo=0,
                hi=10_000_000,
                seen=seen,
                price_cap_per_bucket=500,
                max_pages_per_bucket=50,
                concurrency=2,
            )

    assert len(call_ranges) >= 3, f"Expected >=3 probes, got {call_ranges}"
    assert call_ranges[0] == (None, 10_000_000)  # lo=0 -> _lo_param=None


@pytest.mark.asyncio
async def test_walk_price_range_fallback_when_total_none():
    """totalItems absent in pager triggers paginate-until-empty fallback."""
    s = YandexRealtyScraper()
    call_count = 0

    async def fake_fetch_page_json(rooms, page, price_min, price_max):
        nonlocal call_count
        call_count += 1
        if call_count == 1:
            # Probe: pager without totalItems
            pager = {"page": 0, "pageSize": 20}
            return _make_gate_payload([_ENTITY_FULL], pager)
        if call_count == 2:
            pager = {"page": 0, "pageSize": 20}
            return _make_gate_payload([dict(_ENTITY_FULL, offerId=f"offer_{call_count}")], pager)
        return _make_gate_payload([])

    seen: dict = {}
    bucket_calls: list = []

    with patch.object(s, "_fetch_page_json", side_effect=fake_fetch_page_json):
        with patch("asyncio.sleep", new_callable=AsyncMock):
            await s._walk_price_range(
                rooms="2",
                lo=0,
                hi=15_000_000,
                seen=seen,
                price_cap_per_bucket=500,
                max_pages_per_bucket=10,
                concurrency=2,
                on_bucket=lambda k, n: bucket_calls.append((k, n)),
            )

    assert len(bucket_calls) >= 1, "on_bucket must fire in fallback path"


@pytest.mark.asyncio
async def test_walk_price_range_fires_on_bucket_for_leaf():
    """on_bucket callback fires once for a leaf bucket with small totalItems."""
    s = YandexRealtyScraper()
    bucket_calls: list = []

    async def fake_fetch_page_json(rooms, page, price_min, price_max):
        pager = {"page": 0, "pageSize": 20, "totalItems": 5, "totalPages": 1}
        return _make_gate_payload([_ENTITY_FULL], pager)

    seen: dict = {}
    with patch.object(s, "_fetch_page_json", side_effect=fake_fetch_page_json):
        with patch("asyncio.sleep", new_callable=AsyncMock):
            await s._walk_price_range(
                rooms="1",
                lo=0,
                hi=20_000_000,
                seen=seen,
                price_cap_per_bucket=500,
                max_pages_per_bucket=5,
                concurrency=2,
                on_bucket=lambda k, n: bucket_calls.append(k),
            )

    assert len(bucket_calls) >= 1


@pytest.mark.asyncio
async def test_walk_price_range_skip_buckets():
    """skip_buckets prevents on_bucket from firing for already-done bucket."""
    s = YandexRealtyScraper()
    on_bucket_calls: list = []

    async def fake_fetch_page_json(rooms, page, price_min, price_max):
        pager = {"page": 0, "pageSize": 20, "totalItems": 10, "totalPages": 1}
        return _make_gate_payload([_ENTITY_FULL], pager)

    # lo=0 -> _lo_param=None -> bucket key uses None
    skip_set = {_combo_label("1", 0, 10_000_000)}
    seen: dict = {}
    with patch.object(s, "_fetch_page_json", side_effect=fake_fetch_page_json):
        with patch("asyncio.sleep", new_callable=AsyncMock):
            await s._walk_price_range(
                rooms="1",
                lo=0,
                hi=10_000_000,
                seen=seen,
                price_cap_per_bucket=500,
                max_pages_per_bucket=5,
                concurrency=2,
                on_bucket=lambda k, n: on_bucket_calls.append(k),
                skip_buckets=skip_set,
            )

    assert skip_set.pop() not in on_bucket_calls


@pytest.mark.asyncio
async def test_fetch_all_secondary_deduplicates_across_rooms():
    """fetch_all_secondary deduplicates offers across room buckets."""
    s = YandexRealtyScraper()

    shared_lot = MagicMock(spec=ScrapedLot)
    shared_lot.source_id = "shared_999"
    shared_lot.source_url = "https://realty.yandex.ru/offer/shared_999/"

    unique_lot_1 = MagicMock(spec=ScrapedLot)
    unique_lot_1.source_id = "unique_001"
    unique_lot_1.source_url = "https://realty.yandex.ru/offer/unique_001/"

    unique_lot_2 = MagicMock(spec=ScrapedLot)
    unique_lot_2.source_id = "unique_002"
    unique_lot_2.source_url = "https://realty.yandex.ru/offer/unique_002/"

    call_count = 0

    async def fake_walk(*, rooms, lo, hi, seen, **kwargs):
        nonlocal call_count
        seen["shared_999"] = shared_lot
        if call_count == 0:
            seen["unique_001"] = unique_lot_1
        else:
            seen["unique_002"] = unique_lot_2
        call_count += 1

    with patch.object(s, "_walk_price_range", side_effect=fake_walk):
        result = await s.fetch_all_secondary(rooms_buckets=["1", "2"])

    ids = {lot.source_id for lot in result}
    assert "shared_999" in ids
    assert "unique_001" in ids
    assert "unique_002" in ids
    assert len(ids) == 3