"""Unit tests for YandexRealtyScraper gate-API JSON parser. Replaced DOM/HTML SERP tests (2026-06-17): new transport uses GET realty.yandex.ru/gate/react-page/get/ (JSON gate-API) instead of HTML SERP. """ import json from unittest.mock import AsyncMock, MagicMock, patch import pytest from app.services.scrapers.base import ScrapedLot from app.services.scrapers.yandex_realty import ( _GATE_MAX_PAGES_CAP, DEFAULT_CITY, DEFAULT_PRICE_RANGES, ROOM_PATH, YandexRealtyScraper, _combo_label, _CurlResponse, _entity_to_lot, _extract_gate_data, _extract_json_from_content, _is_gate_error, _parse_gate_json, ) # --------------------------------------------------------------------------- # Minimal gate-API JSON fixtures (derived from real captured sample 2026-06-17) # pager shape: {page, pageSize, totalItems, totalPages} # --------------------------------------------------------------------------- _ENTITY_FULL: dict = { "offerId": 4740475460451078271, "url": "//realty.yandex.ru/offer/4740475460451078271", "price": { "currency": "RUR", "value": 4500000, "valuePerPart": 119048, }, "area": {"value": 37.8, "unit": "SQUARE_METER"}, "livingSpace": {"value": 20, "unit": "SQUARE_METER"}, "kitchenSpace": {"value": 12.7, "unit": "SQUARE_METER"}, "roomsTotal": 1, "floorsOffered": [8], "floorsTotal": 25, "ceilingHeight": 2.7, "building": { "builtYear": 2016, "buildingType": "MONOLIT", "siteId": 280938, "siteName": "Peremen", }, "location": { "geocoderAddress": "Ekaterinburg, ulitsa Evgenia Savkova, 8", "point": {"latitude": 56.79512, "longitude": 60.49186}, }, "mainImages": [ "//avatars.mds.yandex.net/get-realty-offers/11904018/abc/main", "//avatars.mds.yandex.net/get-realty-offers/14717586/def/main", ], } _ENTITY_STUDIO: dict = { "offerId": 9999999, "url": "//realty.yandex.ru/offer/9999999", "price": {"value": 3200000, "valuePerPart": 114285}, "area": {"value": 28.0}, "livingSpace": None, "kitchenSpace": None, "roomsTotal": None, # studio -> rooms=0 "floorsOffered": [2], "floorsTotal": 9, "ceilingHeight": None, "building": {"builtYear": 2010, "buildingType": "PANEL", "siteId": None, "siteName": None}, "location": { "geocoderAddress": "Ekaterinburg, ulitsa Lenina, 5", "point": {"latitude": 56.838, "longitude": 60.597}, }, "mainImages": [], } _ENTITY_NO_PRICE: dict = { "offerId": 1111, "url": "//realty.yandex.ru/offer/1111", "price": {"value": 0}, "area": {"value": 40.0}, "roomsTotal": 1, "floorsOffered": [3], "floorsTotal": 5, "building": {}, "location": {"point": {"latitude": 56.8, "longitude": 60.6}}, "mainImages": [], } _ENTITY_NO_OFFER_ID: dict = { "offerId": None, "price": {"value": 5000000}, "area": {"value": 50.0}, } # Entity carrying the rich gate-API fields (creationDate, description, author, # predictedPrice, price.trend/previous, location.metroList) — verified live 2026-06-17. _ENTITY_RICH_OWNER: dict = { "offerId": 5500000000000000001, "url": "//realty.yandex.ru/offer/5500000000000000001", "creationDate": "2026-05-24T15:10:27Z", "description": "Продаётся светлая квартира с видом на парк.", "author": {"category": "OWNER", "agentName": "Иван", "organization": None}, "price": { "value": 7200000, "valuePerPart": 150000, "trend": "DECREASED", "previous": 7500000, }, # Real gate-API nests valuation under predictions.predictedPrice (verified # live 2026-06-18) — NOT predictedPrice.predictedPrice. "predictions": {"predictedPrice": {"value": "7554000", "min": "7100000", "max": "8000000"}}, "area": {"value": 48.0}, "roomsTotal": 2, "floorsOffered": [5], "floorsTotal": 12, "building": {"builtYear": 2018, "buildingType": "MONOLIT"}, "location": { "geocoderAddress": "Ekaterinburg, ulitsa Mira, 10", "point": {"latitude": 56.84, "longitude": 60.61}, "metroList": [ {"name": "Ploshchad 1905 goda", "timeToMetro": 7}, {"name": "Geologicheskaya", "timeToMetro": 12}, ], }, "mainImages": [], } _ENTITY_RICH_AGENCY: dict = { "offerId": 5500000000000000002, "url": "//realty.yandex.ru/offer/5500000000000000002", "creationDate": "2026-06-01T09:00:00Z", "description": "Агентская продажа.", "author": {"category": "AGENCY", "agentName": "Петров", "organization": "Агентство «Диал»"}, "price": {"value": 9900000, "trend": "UNCHANGED"}, "area": {"value": 60.0}, "roomsTotal": 3, "floorsOffered": [3], "floorsTotal": 9, "building": {"builtYear": 2012, "buildingType": "BRICK"}, "location": { "geocoderAddress": "Ekaterinburg, ulitsa Lenina, 1", "point": {"latitude": 56.83, "longitude": 60.6}, "metro": {"name": "Dinamo", "timeToMetro": 4}, }, "mainImages": [], } def _make_gate_payload(entities: list, pager: dict | None = None) -> dict: """Wrap entities + pager into gate-API response shape.""" if pager is None: pager = {"page": 0, "pageSize": 20, "totalItems": len(entities), "totalPages": 1} return { "response": { "search": { "offers": { "entities": entities, "pager": pager, } } } } # --------------------------------------------------------------------------- # PART A: pure helper functions # --------------------------------------------------------------------------- def test_is_gate_error_on_error_key(): assert _is_gate_error({"error": "bad request"}) is True def test_is_gate_error_on_missing_response(): assert _is_gate_error({"something": "else"}) is True def test_is_gate_error_on_valid_payload(): payload = _make_gate_payload([_ENTITY_FULL]) assert _is_gate_error(payload) is False def test_extract_gate_data_returns_entities_and_pager(): payload = _make_gate_payload([_ENTITY_FULL, _ENTITY_STUDIO]) result = _extract_gate_data(payload) assert result is not None entities, pager = result assert len(entities) == 2 assert "totalItems" in pager def test_extract_gate_data_returns_none_on_bad_shape(): assert _extract_gate_data({}) is None assert _extract_gate_data({"response": {}}) is None assert _extract_gate_data({"response": {"search": {}}}) is None # --------------------------------------------------------------------------- # PART B: _entity_to_lot field mapping # --------------------------------------------------------------------------- def test_entity_to_lot_full_entity(): lot = _entity_to_lot(_ENTITY_FULL) assert lot is not None assert lot.source == "yandex" assert lot.source_id == "4740475460451078271" assert lot.source_url == "https://realty.yandex.ru/offer/4740475460451078271" assert lot.price_rub == 4_500_000 assert lot.price_per_m2 == 119_048 assert lot.area_m2 == pytest.approx(37.8) assert lot.living_area_m2 == pytest.approx(20.0) assert lot.rooms == 1 assert lot.floor == 8 assert lot.total_floors == 25 assert lot.year_built == 2016 assert lot.house_type == "monolith" # normalize_house_type(MONOLIT) — #2007 assert lot.kitchen_area_m2 == pytest.approx(12.7) # promoted to column — #2007 assert lot.ceiling_height_m == pytest.approx(2.7) # promoted to column — #2007 assert lot.lat == pytest.approx(56.79512) assert lot.lon == pytest.approx(60.49186) assert lot.address == "Ekaterinburg, ulitsa Evgenia Savkova, 8" assert lot.house_source == "yandex_realty_nb" assert lot.house_ext_id == "280938" assert lot.listing_segment == "vtorichka" assert len(lot.photo_urls) == 2 assert all(u.startswith("https://avatars.mds.yandex.net") for u in lot.photo_urls) assert lot.raw_payload is not None assert lot.raw_payload["ceiling_height"] == pytest.approx(2.7) assert lot.raw_payload["kitchen_area_m2"] == pytest.approx(12.7) assert lot.raw_payload["raw_building_type"] == "MONOLIT" # сырой SCREAMING сохранён def test_entity_to_lot_ceiling_out_of_range_dropped(): """ceiling_height — numeric(3,2) (max 9.99); неправдоподобный yandex SERP мусор (видели 18 м) отсекаем в None, иначе DataError уронил бы весь батч. Сырое значение остаётся в raw_payload. (#2007)""" lot_hi = _entity_to_lot({**_ENTITY_FULL, "ceilingHeight": 18}) assert lot_hi is not None assert lot_hi.ceiling_height_m is None assert lot_hi.raw_payload["ceiling_height"] == 18 # сырое сохранено lot_lo = _entity_to_lot({**_ENTITY_FULL, "ceilingHeight": 1.6}) assert lot_lo is not None assert lot_lo.ceiling_height_m is None def test_entity_to_lot_segment_vtorichka_default(): """Default new_flat='NO' -> listing_segment='vtorichka'.""" lot = _entity_to_lot(_ENTITY_FULL) assert lot is not None assert lot.listing_segment == "vtorichka" def test_entity_to_lot_segment_novostroyki_on_new_flat_yes(): """new_flat='YES' -> listing_segment='novostroyki'.""" lot = _entity_to_lot(_ENTITY_FULL, new_flat="YES") assert lot is not None assert lot.listing_segment == "novostroyki" def test_parse_gate_json_propagates_segment(): """_parse_gate_json threads new_flat down to each lot's listing_segment.""" payload = _make_gate_payload([_ENTITY_FULL, _ENTITY_STUDIO]) lots_nb = _parse_gate_json(payload, new_flat="YES") assert lots_nb and all(lot.listing_segment == "novostroyki" for lot in lots_nb) lots_vt = _parse_gate_json(payload, new_flat="NO") assert lots_vt and all(lot.listing_segment == "vtorichka" for lot in lots_vt) def test_entity_to_lot_studio_rooms_zero(): lot = _entity_to_lot(_ENTITY_STUDIO) assert lot is not None assert lot.rooms == 0 assert lot.area_m2 == pytest.approx(28.0) assert lot.floor == 2 assert lot.total_floors == 9 assert lot.house_source is None assert lot.house_ext_id is None assert lot.photo_urls == [] def test_entity_to_lot_no_price_returns_none(): lot = _entity_to_lot(_ENTITY_NO_PRICE) assert lot is None def test_entity_to_lot_no_offer_id_returns_none(): lot = _entity_to_lot(_ENTITY_NO_OFFER_ID) assert lot is None def test_entity_to_lot_photo_url_prefix(): """mainImages starting with // must get https: prepended.""" lot = _entity_to_lot(_ENTITY_FULL) assert lot is not None # Both URLs must start with https:// (not protocol-relative //) assert all(u.startswith("https://") for u in lot.photo_urls) assert not any(u.startswith("//") for u in lot.photo_urls) def test_entity_to_lot_photo_capped_at_5(): entity = dict(_ENTITY_FULL) entity["mainImages"] = [f"//avatars.mds.yandex.net/img{i}/main" for i in range(10)] lot = _entity_to_lot(entity) assert lot is not None assert len(lot.photo_urls) == 5 def test_entity_to_lot_url_fallback_when_missing(): """When url field absent, fall back to constructed /offer// URL.""" entity = dict(_ENTITY_FULL) entity["url"] = None lot = _entity_to_lot(entity) assert lot is not None expected_id = _ENTITY_FULL["offerId"] assert lot.source_url == f"https://realty.yandex.ru/offer/{expected_id}/" def test_entity_to_lot_no_house_when_site_id_absent(): """Entity without siteId produces house_source=None, house_ext_id=None.""" entity = dict(_ENTITY_FULL) entity["building"] = {"builtYear": 2015, "buildingType": "BRICK", "siteId": None} lot = _entity_to_lot(entity) assert lot is not None assert lot.house_source is None assert lot.house_ext_id is None def test_entity_to_lot_page_param_in_raw(): """page_param passed to _entity_to_lot appears in raw_payload.""" lot = _entity_to_lot(_ENTITY_FULL, page_param=3) assert lot is not None assert lot.raw_payload["page_param"] == 3 # --------------------------------------------------------------------------- # PART B2: rich gate-API fields (predictedPrice, seller, dates, trend, metro) # --------------------------------------------------------------------------- def test_entity_to_lot_rich_owner_fields(): lot = _entity_to_lot(_ENTITY_RICH_OWNER) assert lot is not None # creationDate → listing_date / publish_date / days_on_market assert lot.listing_date is not None assert lot.listing_date.isoformat() == "2026-05-24" assert lot.publish_date == lot.listing_date assert lot.days_on_market is not None assert lot.days_on_market >= 0 # description assert lot.description == "Продаётся светлая квартира с видом на парк." # author OWNER assert lot.is_homeowner is True assert lot.is_pro_seller is None assert lot.agency_name is None # predictedPrice assert lot.predicted_price_rub == 7_554_000 assert lot.predicted_price_min == 7_100_000 assert lot.predicted_price_max == 8_000_000 # price trend / previous assert lot.price_trend == "DECREASED" assert lot.price_previous_rub == 7_500_000 # metro_stations from metroList assert lot.metro_stations == [ {"name": "Ploshchad 1905 goda", "min": 7}, {"name": "Geologicheskaya", "min": 12}, ] # yandex_offer_id assert lot.yandex_offer_id == "5500000000000000001" def test_entity_to_lot_rich_agency_fields(): lot = _entity_to_lot(_ENTITY_RICH_AGENCY) assert lot is not None # AGENCY → pro seller + agency_name from organization assert lot.is_pro_seller is True assert lot.is_homeowner is None assert lot.agency_name == "Агентство «Диал»" # price.trend present, no previous → price_previous_rub None assert lot.price_trend == "UNCHANGED" assert lot.price_previous_rub is None # no predictedPrice path → all None assert lot.predicted_price_rub is None assert lot.predicted_price_min is None assert lot.predicted_price_max is None # single location.metro fallback assert lot.metro_stations == [{"name": "Dinamo", "min": 4}] def test_entity_to_lot_no_rich_fields_are_none(): """Base fixture lacks rich fields → all new fields default to None/[] (no crash).""" lot = _entity_to_lot(_ENTITY_FULL) assert lot is not None assert lot.listing_date is None assert lot.days_on_market is None assert lot.description is None assert lot.is_homeowner is None assert lot.is_pro_seller is None assert lot.agency_name is None assert lot.predicted_price_rub is None assert lot.price_trend is None assert lot.price_previous_rub is None assert lot.metro_stations == [] # yandex_offer_id is always populated from offerId assert lot.yandex_offer_id == "4740475460451078271" def test_entity_to_lot_bad_creation_date_is_safe(): """Unparseable creationDate → listing_date/days_on_market None, no exception.""" entity = dict(_ENTITY_RICH_OWNER) entity["creationDate"] = "not-a-date" lot = _entity_to_lot(entity) assert lot is not None assert lot.listing_date is None assert lot.days_on_market is None # --------------------------------------------------------------------------- # PART C: _parse_gate_json (list-level) # --------------------------------------------------------------------------- def test_parse_gate_json_returns_list(): payload = _make_gate_payload([_ENTITY_FULL, _ENTITY_STUDIO]) lots = _parse_gate_json(payload) assert len(lots) == 2 assert all(isinstance(lot, ScrapedLot) for lot in lots) def test_parse_gate_json_skips_invalid(): """Entities that fail validation (no price, no id) are silently skipped.""" payload = _make_gate_payload([_ENTITY_FULL, _ENTITY_NO_PRICE, _ENTITY_NO_OFFER_ID]) lots = _parse_gate_json(payload) assert len(lots) == 1 assert lots[0].source_id == str(_ENTITY_FULL["offerId"]) def test_parse_gate_json_empty_entities(): payload = _make_gate_payload([]) lots = _parse_gate_json(payload) assert lots == [] def test_parse_gate_json_bad_payload(): """Error/missing-path payload returns empty list without raising.""" assert _parse_gate_json({}) == [] assert _parse_gate_json({"error": "bad"}) == [] # --------------------------------------------------------------------------- # PART D: _build_url gate-API URL shape # --------------------------------------------------------------------------- def test_build_url_contains_required_params(): s = YandexRealtyScraper() url = s._build_url(page=1) assert "realty.yandex.ru/gate/react-page/get/" in url assert "rgid=559132" in url assert "type=SELL" in url assert "category=APARTMENT" in url assert "newFlat=NO" in url assert "_pageType=search" in url assert "_providers=react-search-results-data" in url assert "page=1" in url def test_build_url_page_1_based(): """page param is 1-based; page=1 is first page.""" s = YandexRealtyScraper() url1 = s._build_url(page=1) url2 = s._build_url(page=2) assert "page=1" in url1 assert "page=2" in url2 def test_build_url_rooms_studio(): s = YandexRealtyScraper() url = s._build_url(page=1, rooms="studio") assert "roomsTotal=STUDIO" in url def test_build_url_rooms_1(): s = YandexRealtyScraper() url = s._build_url(page=1, rooms="1") assert "roomsTotal=1" in url def test_build_url_rooms_4plus(): s = YandexRealtyScraper() url = s._build_url(page=1, rooms="4+") assert "roomsTotal=4" in url def test_build_url_price_range(): s = YandexRealtyScraper() url = s._build_url(page=1, price_min=5_000_000, price_max=7_000_000) assert "priceMin=5000000" in url assert "priceMax=7000000" in url def test_build_url_unknown_rooms_no_rooms_param(): """Unknown rooms key must not produce a roomsTotal param.""" s = YandexRealtyScraper() url = s._build_url(page=1, rooms="99-room-penthouse") assert "roomsTotal" not in url def test_build_url_combos_are_unique(): """All rooms x price-range combos produce distinct URLs.""" s = YandexRealtyScraper() urls = set() for rooms in ROOM_PATH.keys(): for lo, hi in DEFAULT_PRICE_RANGES: urls.add(s._build_url(page=1, rooms=rooms, price_min=lo, price_max=hi)) expected = len(ROOM_PATH) * len(DEFAULT_PRICE_RANGES) assert len(urls) == expected, f"URL collisions: {expected} combos but {len(urls)} unique" def test_build_url_new_flat_default_no(): """Default new_flat must keep newFlat=NO (vtorichka) — backward compat.""" s = YandexRealtyScraper() url = s._build_url(page=1) assert "newFlat=NO" in url assert "newFlat=YES" not in url def test_build_url_new_flat_yes(): """new_flat='YES' must produce newFlat=YES (novostroyki segment).""" s = YandexRealtyScraper() url = s._build_url(page=1, new_flat="YES") assert "newFlat=YES" in url def test_default_city(): assert DEFAULT_CITY == "ekaterinburg" def test_gate_max_pages_cap(): assert _GATE_MAX_PAGES_CAP == 50 # --------------------------------------------------------------------------- # PART E: _combo_label # --------------------------------------------------------------------------- def test_combo_label_none_none(): result = _combo_label(None, 0, 200_000_000) assert isinstance(result, str) assert "any" in result def test_combo_label_room_and_price(): result = _combo_label("1", 5_000_000, 7_000_000) assert "1" in result assert "5000000" in result assert "7000000" in result def test_combo_label_different_ranges_differ(): k1 = _combo_label("1", 5_000_000, 7_000_000) k2 = _combo_label("1", 7_000_000, 10_000_000) k3 = _combo_label("2", 5_000_000, 7_000_000) assert k1 != k2 assert k1 != k3 # --------------------------------------------------------------------------- # PART F: transport layer # --------------------------------------------------------------------------- # camoufox renders the gate-API JSON wrapped in an HTML
 (verified prod 2026-06-17):
_CAMOUFOX_HTML_PREFIX = (
    '
'
)
_CAMOUFOX_HTML_SUFFIX = "
" def _wrap_camoufox(json_text: str) -> str: """Wrap a JSON string the way camoufox renders a JSON endpoint.""" return f"{_CAMOUFOX_HTML_PREFIX}{json_text}{_CAMOUFOX_HTML_SUFFIX}" def test_extract_json_from_content_unwraps_camoufox_pre(): """_extract_json_from_content must unwrap camoufox's
{json}
HTML. Feeds the real prod-observed wrapper and asserts the extracted text parses back to the original gate-API payload. """ payload = _make_gate_payload([_ENTITY_FULL]) wrapped = _wrap_camoufox(json.dumps(payload)) extracted = _extract_json_from_content(wrapped) assert extracted is not None assert extracted.startswith('{"response"') assert json.loads(extracted) == payload def test_extract_json_from_content_raw_json_fallback(): """Raw JSON (no HTML wrapper) is returned via the first-'{' fallback.""" raw = json.dumps(_make_gate_payload([])) assert _extract_json_from_content(raw) == raw def test_extract_json_from_content_returns_none_when_no_json(): assert _extract_json_from_content("nope") is None @pytest.mark.asyncio async def test_aexit_closes_browser_and_session(): s = YandexRealtyScraper() mock_browser = AsyncMock() mock_session = AsyncMock() s._browser = mock_browser s._cffi_session = mock_session await s.__aexit__(None, None, None) mock_browser.__aexit__.assert_awaited_once() mock_session.close.assert_awaited_once() assert s._browser is None assert s._cffi_session is None @pytest.mark.asyncio async def test_http_get_raises_when_no_browser(): """_http_get must raise RuntimeError outside async context manager (no browser).""" s = YandexRealtyScraper() with pytest.raises(RuntimeError, match="async context manager"): await s._http_get("https://realty.yandex.ru/test/") @pytest.mark.asyncio async def test_http_get_unwraps_camoufox_html_to_json_body(): """_http_get fetches via BrowserFetcher and returns extracted JSON with status 200.""" s = YandexRealtyScraper() payload = _make_gate_payload([_ENTITY_FULL]) wrapped = _wrap_camoufox(json.dumps(payload)) mock_browser = AsyncMock() mock_browser.fetch = AsyncMock(return_value=wrapped) s._browser = mock_browser resp = await s._http_get("https://realty.yandex.ru/test/", timeout=30) assert isinstance(resp, _CurlResponse) assert resp.status_code == 200 assert json.loads(resp.text) == payload mock_browser.fetch.assert_awaited_once_with("https://realty.yandex.ru/test/") @pytest.mark.asyncio async def test_http_get_status_0_on_fetch_exception(): """BrowserFetcher.fetch raising -> _CurlResponse(status_code=0) (transient failure).""" s = YandexRealtyScraper() mock_browser = AsyncMock() mock_browser.fetch = AsyncMock(side_effect=RuntimeError("browser down")) s._browser = mock_browser resp = await s._http_get("https://realty.yandex.ru/test/") assert resp.status_code == 0 assert resp.text == "" @pytest.mark.asyncio async def test_http_get_status_0_when_no_json_in_content(): """Content with no JSON blob -> _CurlResponse(status_code=0).""" s = YandexRealtyScraper() mock_browser = AsyncMock() mock_browser.fetch = AsyncMock(return_value="blocked") s._browser = mock_browser resp = await s._http_get("https://realty.yandex.ru/test/") assert resp.status_code == 0 # --------------------------------------------------------------------------- # PART G: fetch_around — tarpit resilience # --------------------------------------------------------------------------- def _valid_gate_text(entity: dict | None = None) -> str: ent = entity or _ENTITY_FULL pager = {"page": 0, "pageSize": 20, "totalItems": 1, "totalPages": 1} return json.dumps(_make_gate_payload([ent], pager)) @pytest.mark.asyncio async def test_fetch_around_returns_lots_on_200_json(): s = YandexRealtyScraper() body = _valid_gate_text() async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse: return _CurlResponse(status_code=200, text=body) with patch.object(s, "_http_get", side_effect=fake_http_get): with patch.object(s, "sleep_between_requests", new_callable=AsyncMock): lots = await s.fetch_around(lat=56.8, lon=60.6, rooms="1") assert len(lots) == 1 assert lots[0].source_id == str(_ENTITY_FULL["offerId"]) @pytest.mark.asyncio async def test_fetch_around_rotates_and_retries_on_status_0(): """fetch_around rotates IP and retries on status_code==0 (tarpit).""" s = YandexRealtyScraper() call_count = 0 body = _valid_gate_text() async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse: nonlocal call_count call_count += 1 if call_count == 1: return _CurlResponse(status_code=0, text="") return _CurlResponse(status_code=200, text=body) with patch.object(s, "_http_get", side_effect=fake_http_get): with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate: with patch("asyncio.sleep", new_callable=AsyncMock): with patch.object(s, "sleep_between_requests", new_callable=AsyncMock): lots = await s.fetch_around(lat=56.84, lon=60.60, rooms="1") assert mock_rotate.call_count == 1 assert call_count == 2 assert len(lots) >= 1 @pytest.mark.asyncio async def test_fetch_around_rotates_on_json_error(): """fetch_around rotates when response body is not valid JSON.""" s = YandexRealtyScraper() call_count = 0 body = _valid_gate_text() async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse: nonlocal call_count call_count += 1 if call_count == 1: return _CurlResponse(status_code=200, text="not json") return _CurlResponse(status_code=200, text=body) with patch.object(s, "_http_get", side_effect=fake_http_get): with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate: with patch("asyncio.sleep", new_callable=AsyncMock): with patch.object(s, "sleep_between_requests", new_callable=AsyncMock): lots = await s.fetch_around(lat=56.84, lon=60.60, rooms="2") assert mock_rotate.call_count == 1 assert len(lots) >= 1 @pytest.mark.asyncio async def test_fetch_around_exhausts_retries_returns_empty(): """fetch_around returns [] when all retries exhausted.""" s = YandexRealtyScraper() async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse: return _CurlResponse(status_code=0, text="") with patch.object(s, "_http_get", side_effect=fake_http_get): with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate: with patch("asyncio.sleep", new_callable=AsyncMock): lots = await s.fetch_around(lat=56.84, lon=60.60, rooms="1") assert mock_rotate.call_count >= 1 assert lots == [] @pytest.mark.asyncio async def test_fetch_around_no_rotate_on_404(): """fetch_around does NOT rotate on 404 -- only on tarpit (status==0) / bad JSON.""" s = YandexRealtyScraper() async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse: return _CurlResponse(status_code=404, text="Not Found") with patch.object(s, "_http_get", side_effect=fake_http_get): with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate: lots = await s.fetch_around(lat=56.84, lon=60.60) assert mock_rotate.call_count == 0 assert lots == [] # --------------------------------------------------------------------------- # PART H: fetch_around_multi_room # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_fetch_around_multi_room_dedup(): """Multi-room fetch deduplicates offers seen in multiple combos. fetch_around_multi_room fetches page=1 inline via _http_get, not via fetch_around. We mock _http_get to return JSON payloads with shared/unique offer IDs to verify deduplication. """ s = YandexRealtyScraper() # Combo 0: shared_offer + unique_0 # Combo 1: shared_offer + unique_1 entity_shared = dict( _ENTITY_FULL, offerId="shared_offer_1234", url="//realty.yandex.ru/offer/shared_offer_1234" ) entity_unique_0 = dict( _ENTITY_FULL, offerId="unique_offer_0", url="//realty.yandex.ru/offer/unique_offer_0" ) entity_unique_1 = dict( _ENTITY_FULL, offerId="unique_offer_1", url="//realty.yandex.ru/offer/unique_offer_1" ) pager = {"page": 0, "pageSize": 20, "totalItems": 2, "totalPages": 1} payload_combo0 = _make_gate_payload([entity_shared, entity_unique_0], pager) payload_combo1 = _make_gate_payload([entity_shared, entity_unique_1], pager) combo_payloads = [payload_combo0, payload_combo1] call_idx = 0 async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse: nonlocal call_idx idx = call_idx call_idx += 1 if idx < len(combo_payloads): return _CurlResponse(status_code=200, text=json.dumps(combo_payloads[idx])) return _CurlResponse(status_code=200, text=json.dumps(_make_gate_payload([]))) mock_settings = MagicMock() mock_settings.yandex_proxy_url = None with patch.dict("sys.modules", {"app.core.config": MagicMock(settings=mock_settings)}): with patch.object(s, "_http_get", side_effect=fake_http_get): with patch.object(s, "sleep_between_requests", new_callable=AsyncMock): lots = await s.fetch_around_multi_room( lat=0.0, lon=0.0, rooms_list=["1"], price_ranges=[(None, 5_000_000), (5_000_000, 7_000_000)], max_pages=1, ) ids = {lot.source_id for lot in lots} assert "shared_offer_1234" in ids assert len(ids) == 3 @pytest.mark.asyncio async def test_fetch_around_multi_room_runs_both_segments_with_shared_dedup(): """segments=['NO','YES'] sweeps both vtorichka+novostroyki; dedup spans both passes. Mocks _http_get and inspects requested URLs: both newFlat=NO and newFlat=YES must be requested. The same offerId returned in both passes must be deduped to one lot (single shared `seen` across passes), tagged by the FIRST pass's segment. """ s = YandexRealtyScraper() requested_new_flat: list[str] = [] entity_shared = dict( _ENTITY_FULL, offerId="shared_xseg", url="//realty.yandex.ru/offer/shared_xseg" ) entity_vt = dict(_ENTITY_FULL, offerId="vt_only", url="//realty.yandex.ru/offer/vt_only") entity_nb = dict(_ENTITY_FULL, offerId="nb_only", url="//realty.yandex.ru/offer/nb_only") pager = {"page": 0, "pageSize": 20, "totalItems": 2, "totalPages": 1} async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse: if "newFlat=YES" in url: requested_new_flat.append("YES") payload = _make_gate_payload([entity_shared, entity_nb], pager) else: requested_new_flat.append("NO") payload = _make_gate_payload([entity_shared, entity_vt], pager) return _CurlResponse(status_code=200, text=json.dumps(payload)) with patch.object(s, "_http_get", side_effect=fake_http_get): with patch.object(s, "sleep_between_requests", new_callable=AsyncMock): lots = await s.fetch_around_multi_room( lat=0.0, lon=0.0, rooms_list=["1"], price_ranges=[(None, 5_000_000)], segments=["NO", "YES"], max_pages=1, ) assert "NO" in requested_new_flat assert "YES" in requested_new_flat by_id = {lot.source_id: lot for lot in lots} assert set(by_id) == {"shared_xseg", "vt_only", "nb_only"} # Shared offer deduped to ONE lot, tagged by the first pass (NO -> vtorichka). assert by_id["shared_xseg"].listing_segment == "vtorichka" assert by_id["vt_only"].listing_segment == "vtorichka" assert by_id["nb_only"].listing_segment == "novostroyki" @pytest.mark.asyncio async def test_fetch_around_multi_room_default_segment_no_only(): """Without segments arg, only newFlat=NO (vtorichka) is requested (legacy default).""" s = YandexRealtyScraper() requested_new_flat: list[str] = [] pager = {"page": 0, "pageSize": 20, "totalItems": 1, "totalPages": 1} async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse: requested_new_flat.append("YES" if "newFlat=YES" in url else "NO") body = json.dumps(_make_gate_payload([_ENTITY_FULL], pager)) return _CurlResponse(status_code=200, text=body) with patch.object(s, "_http_get", side_effect=fake_http_get): with patch.object(s, "sleep_between_requests", new_callable=AsyncMock): await s.fetch_around_multi_room( lat=0.0, lon=0.0, rooms_list=["1"], price_ranges=[(None, 5_000_000)], max_pages=1, ) assert set(requested_new_flat) == {"NO"} @pytest.mark.asyncio async def test_fetch_around_multi_room_legacy_single_sweep(): """Legacy mode (no rooms_list/price_ranges) uses single citywide sweep.""" s = YandexRealtyScraper() calls: list[dict] = [] async def fake_fetch_around( lat, lon, radius_m=1000, page=1, rooms=None, price_min=None, price_max=None ): calls.append({"page": page, "rooms": rooms, "price_min": price_min}) return [] with patch.object(s, "fetch_around", side_effect=fake_fetch_around): await s.fetch_around_multi_room(lat=0.0, lon=0.0, max_pages=3) assert all(c["rooms"] is None for c in calls) assert all(c["price_min"] is None for c in calls) # --------------------------------------------------------------------------- # PART I: exhaustive load -- fetch_all_secondary + _walk_price_range # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_walk_price_range_splits_when_total_exceeds_cap(): """When totalItems > cap, _walk_price_range splits into sub-ranges recursively.""" s = YandexRealtyScraper() call_ranges: list[tuple] = [] call_count = 0 async def fake_fetch_page_json(rooms, page, price_min, price_max): nonlocal call_count call_count += 1 call_ranges.append((price_min, price_max)) # First call (wide range): 600 items -> triggers split total = 600 if call_count == 1 else 10 entities = [dict(_ENTITY_FULL, offerId=f"offer_{call_count}")] pager = { "page": 0, "pageSize": 20, "totalItems": total, "totalPages": max(1, total // 20), } return _make_gate_payload(entities, pager) seen: dict = {} with patch.object(s, "_fetch_page_json", side_effect=fake_fetch_page_json): with patch("asyncio.sleep", new_callable=AsyncMock): await s._walk_price_range( rooms="1", lo=0, hi=10_000_000, seen=seen, price_cap_per_bucket=500, max_pages_per_bucket=50, concurrency=2, ) assert len(call_ranges) >= 3, f"Expected >=3 probes, got {call_ranges}" assert call_ranges[0] == (None, 10_000_000) # lo=0 -> _lo_param=None @pytest.mark.asyncio async def test_walk_price_range_fallback_when_total_none(): """totalItems absent in pager triggers paginate-until-empty fallback.""" s = YandexRealtyScraper() call_count = 0 async def fake_fetch_page_json(rooms, page, price_min, price_max): nonlocal call_count call_count += 1 if call_count == 1: # Probe: pager without totalItems pager = {"page": 0, "pageSize": 20} return _make_gate_payload([_ENTITY_FULL], pager) if call_count == 2: pager = {"page": 0, "pageSize": 20} return _make_gate_payload([dict(_ENTITY_FULL, offerId=f"offer_{call_count}")], pager) return _make_gate_payload([]) seen: dict = {} bucket_calls: list = [] with patch.object(s, "_fetch_page_json", side_effect=fake_fetch_page_json): with patch("asyncio.sleep", new_callable=AsyncMock): await s._walk_price_range( rooms="2", lo=0, hi=15_000_000, seen=seen, price_cap_per_bucket=500, max_pages_per_bucket=10, concurrency=2, on_bucket=lambda k, n: bucket_calls.append((k, n)), ) assert len(bucket_calls) >= 1, "on_bucket must fire in fallback path" @pytest.mark.asyncio async def test_walk_price_range_fires_on_bucket_for_leaf(): """on_bucket callback fires once for a leaf bucket with small totalItems.""" s = YandexRealtyScraper() bucket_calls: list = [] async def fake_fetch_page_json(rooms, page, price_min, price_max): pager = {"page": 0, "pageSize": 20, "totalItems": 5, "totalPages": 1} return _make_gate_payload([_ENTITY_FULL], pager) seen: dict = {} with patch.object(s, "_fetch_page_json", side_effect=fake_fetch_page_json): with patch("asyncio.sleep", new_callable=AsyncMock): await s._walk_price_range( rooms="1", lo=0, hi=20_000_000, seen=seen, price_cap_per_bucket=500, max_pages_per_bucket=5, concurrency=2, on_bucket=lambda k, n: bucket_calls.append(k), ) assert len(bucket_calls) >= 1 @pytest.mark.asyncio async def test_walk_price_range_skip_buckets(): """skip_buckets prevents on_bucket from firing for already-done bucket.""" s = YandexRealtyScraper() on_bucket_calls: list = [] async def fake_fetch_page_json(rooms, page, price_min, price_max): pager = {"page": 0, "pageSize": 20, "totalItems": 10, "totalPages": 1} return _make_gate_payload([_ENTITY_FULL], pager) # lo=0 -> _lo_param=None -> bucket key uses None skip_set = {_combo_label("1", 0, 10_000_000)} seen: dict = {} with patch.object(s, "_fetch_page_json", side_effect=fake_fetch_page_json): with patch("asyncio.sleep", new_callable=AsyncMock): await s._walk_price_range( rooms="1", lo=0, hi=10_000_000, seen=seen, price_cap_per_bucket=500, max_pages_per_bucket=5, concurrency=2, on_bucket=lambda k, n: on_bucket_calls.append(k), skip_buckets=skip_set, ) assert skip_set.pop() not in on_bucket_calls @pytest.mark.asyncio async def test_fetch_all_secondary_deduplicates_across_rooms(): """fetch_all_secondary deduplicates offers across room buckets.""" s = YandexRealtyScraper() shared_lot = MagicMock(spec=ScrapedLot) shared_lot.source_id = "shared_999" shared_lot.source_url = "https://realty.yandex.ru/offer/shared_999/" unique_lot_1 = MagicMock(spec=ScrapedLot) unique_lot_1.source_id = "unique_001" unique_lot_1.source_url = "https://realty.yandex.ru/offer/unique_001/" unique_lot_2 = MagicMock(spec=ScrapedLot) unique_lot_2.source_id = "unique_002" unique_lot_2.source_url = "https://realty.yandex.ru/offer/unique_002/" call_count = 0 async def fake_walk(*, rooms, lo, hi, seen, **kwargs): nonlocal call_count seen["shared_999"] = shared_lot if call_count == 0: seen["unique_001"] = unique_lot_1 else: seen["unique_002"] = unique_lot_2 call_count += 1 with patch.object(s, "_walk_price_range", side_effect=fake_walk): result = await s.fetch_all_secondary(rooms_buckets=["1", "2"]) ids = {lot.source_id for lot in result} assert "shared_999" in ids assert "unique_001" in ids assert "unique_002" in ids assert len(ids) == 3 @pytest.mark.asyncio async def test_fetch_all_secondary_walks_shared_seed_grid(): """fetch_all_secondary итерирует общую seed-сетку: len(_SEED) walk-вызовов/комнатность. Закрытые брекеты передают hi = br_hi - 1; открытый (последний) — hi=None. """ from app.services.scrapers.price_brackets import get_price_seed_brackets seed = get_price_seed_brackets() s = YandexRealtyScraper() walk_args: list[tuple[int, int | None]] = [] async def fake_walk(*, rooms, lo, hi, seen, **kwargs): walk_args.append((lo, hi)) with patch.object(s, "_walk_price_range", side_effect=fake_walk): await s.fetch_all_secondary(rooms_buckets=["1"]) assert len(walk_args) == len( seed ), f"Ожидалось {len(seed)} walk-вызовов (seed-брекеты), получено {len(walk_args)}" # Первый брекет закрыт: hi = seed[0].hi - 1. first_lo, first_hi = seed[0] assert first_hi is not None assert walk_args[0] == (first_lo, first_hi - 1) # Последний брекет открыт: hi=None (без priceMax-потолка). last_lo, last_hi = seed[-1] assert last_hi is None assert walk_args[-1] == (last_lo, None) @pytest.mark.asyncio async def test_walk_price_range_open_bracket_no_split(): """Открытый брекет (hi=None) не делится — пагинируется leaf напрямую (one bucket). totalItems > cap, но hi=None → split невозможен → ровно один leaf-бакет. """ s = YandexRealtyScraper() bucket_calls: list = [] seen_prices: list[int | None] = [] async def fake_fetch_page_json(rooms, page, price_min, price_max): seen_prices.append(price_max) # total > cap, но 1 страница лотов → leaf пагинируется без split pager = {"page": 0, "pageSize": 20, "totalItems": 9999, "totalPages": 1} return _make_gate_payload([_ENTITY_FULL], pager) seen: dict = {} with patch.object(s, "_fetch_page_json", side_effect=fake_fetch_page_json): with patch("asyncio.sleep", new_callable=AsyncMock): await s._walk_price_range( rooms="1", lo=250_000_000, hi=None, seen=seen, price_cap_per_bucket=500, max_pages_per_bucket=5, concurrency=2, on_bucket=lambda k, n: bucket_calls.append(k), ) # Открытый брекет: НЕ делится (один on_bucket), priceMax всегда None (без потолка). assert len(bucket_calls) == 1, f"Открытый брекет не должен делиться, got {bucket_calls}" assert all( pm is None for pm in seen_prices ), f"Открытый брекет: priceMax всегда None, got {seen_prices}" # --------------------------------------------------------------------------- # PART J: fetch_around_multi_room on_combo callback (FIX A — incremental save) # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_fetch_around_multi_room_on_combo_called_per_combo(): """on_combo вызывается после каждого combo с новыми (деduped) лотами. 2 combo (rooms_list=['1'], 2 price_ranges): каждый имеет уникальный лот. on_combo должен быть вызван дважды — по одному разу на combo. """ s = YandexRealtyScraper() entity_0 = dict(_ENTITY_FULL, offerId="combo0_lot", url="//realty.yandex.ru/offer/combo0_lot") entity_1 = dict(_ENTITY_FULL, offerId="combo1_lot", url="//realty.yandex.ru/offer/combo1_lot") pager = {"page": 0, "pageSize": 20, "totalItems": 1, "totalPages": 1} call_idx = 0 payloads = [ _make_gate_payload([entity_0], pager), _make_gate_payload([entity_1], pager), ] async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse: nonlocal call_idx idx = call_idx call_idx += 1 if idx < len(payloads): return _CurlResponse(status_code=200, text=json.dumps(payloads[idx])) return _CurlResponse(status_code=200, text=json.dumps(_make_gate_payload([]))) on_combo_calls: list[tuple[str, list]] = [] def on_combo(label: str, new_lots: list) -> None: on_combo_calls.append((label, list(new_lots))) with patch.object(s, "_http_get", side_effect=fake_http_get): with patch.object(s, "sleep_between_requests", new_callable=AsyncMock): lots = await s.fetch_around_multi_room( lat=0.0, lon=0.0, rooms_list=["1"], price_ranges=[(None, 5_000_000), (5_000_000, 10_000_000)], max_pages=1, on_combo=on_combo, ) # Два уникальных лота, два combo. assert len(lots) == 2 # on_combo вызван ровно дважды — по одному на combo. assert len(on_combo_calls) == 2, f"Ожидалось 2 on_combo вызова, получено {len(on_combo_calls)}" # Каждый on_combo получил ровно один новый лот. for _label, combo_lots in on_combo_calls: assert len(combo_lots) == 1 # Конкретные source_id в правильных batch'ах. all_ids = {lot.source_id for _, combo_lots in on_combo_calls for lot in combo_lots} assert all_ids == {"combo0_lot", "combo1_lot"} @pytest.mark.asyncio async def test_fetch_around_multi_room_on_combo_not_called_for_failed_combo(): """on_combo НЕ вызывается для combo, у которого все page-1 retries исчерпаны. FIX B: bounded retry — если status_code=0 на всех попытках page=1, combo пропускается. on_combo не должен быть вызван для пропущенного combo. Успешный combo (второй) — on_combo должен сработать. """ s = YandexRealtyScraper() entity_ok = dict(_ENTITY_FULL, offerId="ok_lot", url="//realty.yandex.ru/offer/ok_lot") pager = {"page": 0, "pageSize": 20, "totalItems": 1, "totalPages": 1} call_idx = 0 async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse: nonlocal call_idx call_idx += 1 # Первые (1 + _YANDEX_TARPIT_MAX_RETRIES) запросы — это retries для combo[0] → fail. from app.services.scrapers.yandex_realty import _YANDEX_TARPIT_MAX_RETRIES if call_idx <= 1 + _YANDEX_TARPIT_MAX_RETRIES: return _CurlResponse(status_code=0, text="") # Следующий запрос — combo[1] page=1 → success. return _CurlResponse( status_code=200, text=json.dumps(_make_gate_payload([entity_ok], pager)) ) on_combo_calls: list[str] = [] def on_combo(label: str, new_lots: list) -> None: on_combo_calls.append(label) with patch.object(s, "_http_get", side_effect=fake_http_get): with patch.object(s, "_rotate_ip", new_callable=AsyncMock): with patch("asyncio.sleep", new_callable=AsyncMock): with patch.object(s, "sleep_between_requests", new_callable=AsyncMock): lots = await s.fetch_around_multi_room( lat=0.0, lon=0.0, rooms_list=["1"], price_ranges=[(None, 5_000_000), (5_000_000, 10_000_000)], max_pages=1, on_combo=on_combo, ) # combo[0] скипнут, combo[1] успешен → 1 лот в итоге. assert len(lots) == 1 assert lots[0].source_id == "ok_lot" # on_combo вызван только для успешного combo, не для скипнутого. assert len(on_combo_calls) == 1, ( f"on_combo должен быть вызван только 1 раз (успешный combo), " f"получено {len(on_combo_calls)}: {on_combo_calls}" )