974 lines
34 KiB
Python
974 lines
34 KiB
Python
"""Unit tests for YandexRealtyScraper gate-API JSON parser.
|
|
|
|
Replaced DOM/HTML SERP tests (2026-06-17): new transport uses
|
|
GET realty.yandex.ru/gate/react-page/get/ (JSON gate-API) instead of HTML SERP.
|
|
"""
|
|
|
|
import json
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from app.services.scrapers.base import ScrapedLot
|
|
from app.services.scrapers.yandex_realty import (
|
|
_GATE_MAX_PAGES_CAP,
|
|
DEFAULT_CITY,
|
|
DEFAULT_PRICE_RANGES,
|
|
ROOM_PATH,
|
|
YandexRealtyScraper,
|
|
_combo_label,
|
|
_CurlResponse,
|
|
_entity_to_lot,
|
|
_extract_gate_data,
|
|
_extract_json_from_content,
|
|
_is_gate_error,
|
|
_parse_gate_json,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Minimal gate-API JSON fixtures (derived from real captured sample 2026-06-17)
|
|
# pager shape: {page, pageSize, totalItems, totalPages}
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_ENTITY_FULL: dict = {
|
|
"offerId": 4740475460451078271,
|
|
"url": "//realty.yandex.ru/offer/4740475460451078271",
|
|
"price": {
|
|
"currency": "RUR",
|
|
"value": 4500000,
|
|
"valuePerPart": 119048,
|
|
},
|
|
"area": {"value": 37.8, "unit": "SQUARE_METER"},
|
|
"livingSpace": {"value": 20, "unit": "SQUARE_METER"},
|
|
"kitchenSpace": {"value": 12.7, "unit": "SQUARE_METER"},
|
|
"roomsTotal": 1,
|
|
"floorsOffered": [8],
|
|
"floorsTotal": 25,
|
|
"ceilingHeight": 2.7,
|
|
"building": {
|
|
"builtYear": 2016,
|
|
"buildingType": "MONOLIT",
|
|
"siteId": 280938,
|
|
"siteName": "Peremen",
|
|
},
|
|
"location": {
|
|
"geocoderAddress": "Ekaterinburg, ulitsa Evgenia Savkova, 8",
|
|
"point": {"latitude": 56.79512, "longitude": 60.49186},
|
|
},
|
|
"mainImages": [
|
|
"//avatars.mds.yandex.net/get-realty-offers/11904018/abc/main",
|
|
"//avatars.mds.yandex.net/get-realty-offers/14717586/def/main",
|
|
],
|
|
}
|
|
|
|
_ENTITY_STUDIO: dict = {
|
|
"offerId": 9999999,
|
|
"url": "//realty.yandex.ru/offer/9999999",
|
|
"price": {"value": 3200000, "valuePerPart": 114285},
|
|
"area": {"value": 28.0},
|
|
"livingSpace": None,
|
|
"kitchenSpace": None,
|
|
"roomsTotal": None, # studio -> rooms=0
|
|
"floorsOffered": [2],
|
|
"floorsTotal": 9,
|
|
"ceilingHeight": None,
|
|
"building": {"builtYear": 2010, "buildingType": "PANEL", "siteId": None, "siteName": None},
|
|
"location": {
|
|
"geocoderAddress": "Ekaterinburg, ulitsa Lenina, 5",
|
|
"point": {"latitude": 56.838, "longitude": 60.597},
|
|
},
|
|
"mainImages": [],
|
|
}
|
|
|
|
_ENTITY_NO_PRICE: dict = {
|
|
"offerId": 1111,
|
|
"url": "//realty.yandex.ru/offer/1111",
|
|
"price": {"value": 0},
|
|
"area": {"value": 40.0},
|
|
"roomsTotal": 1,
|
|
"floorsOffered": [3],
|
|
"floorsTotal": 5,
|
|
"building": {},
|
|
"location": {"point": {"latitude": 56.8, "longitude": 60.6}},
|
|
"mainImages": [],
|
|
}
|
|
|
|
_ENTITY_NO_OFFER_ID: dict = {
|
|
"offerId": None,
|
|
"price": {"value": 5000000},
|
|
"area": {"value": 50.0},
|
|
}
|
|
|
|
|
|
def _make_gate_payload(entities: list, pager: dict | None = None) -> dict:
|
|
"""Wrap entities + pager into gate-API response shape."""
|
|
if pager is None:
|
|
pager = {"page": 0, "pageSize": 20, "totalItems": len(entities), "totalPages": 1}
|
|
return {
|
|
"response": {
|
|
"search": {
|
|
"offers": {
|
|
"entities": entities,
|
|
"pager": pager,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# PART A: pure helper functions
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_is_gate_error_on_error_key():
|
|
assert _is_gate_error({"error": "bad request"}) is True
|
|
|
|
|
|
def test_is_gate_error_on_missing_response():
|
|
assert _is_gate_error({"something": "else"}) is True
|
|
|
|
|
|
def test_is_gate_error_on_valid_payload():
|
|
payload = _make_gate_payload([_ENTITY_FULL])
|
|
assert _is_gate_error(payload) is False
|
|
|
|
|
|
def test_extract_gate_data_returns_entities_and_pager():
|
|
payload = _make_gate_payload([_ENTITY_FULL, _ENTITY_STUDIO])
|
|
result = _extract_gate_data(payload)
|
|
assert result is not None
|
|
entities, pager = result
|
|
assert len(entities) == 2
|
|
assert "totalItems" in pager
|
|
|
|
|
|
def test_extract_gate_data_returns_none_on_bad_shape():
|
|
assert _extract_gate_data({}) is None
|
|
assert _extract_gate_data({"response": {}}) is None
|
|
assert _extract_gate_data({"response": {"search": {}}}) is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# PART B: _entity_to_lot field mapping
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_entity_to_lot_full_entity():
|
|
lot = _entity_to_lot(_ENTITY_FULL)
|
|
assert lot is not None
|
|
assert lot.source == "yandex"
|
|
assert lot.source_id == "4740475460451078271"
|
|
assert lot.source_url == "https://realty.yandex.ru/offer/4740475460451078271"
|
|
assert lot.price_rub == 4_500_000
|
|
assert lot.price_per_m2 == 119_048
|
|
assert lot.area_m2 == pytest.approx(37.8)
|
|
assert lot.living_area_m2 == pytest.approx(20.0)
|
|
assert lot.rooms == 1
|
|
assert lot.floor == 8
|
|
assert lot.total_floors == 25
|
|
assert lot.year_built == 2016
|
|
assert lot.house_type == "MONOLIT"
|
|
assert lot.lat == pytest.approx(56.79512)
|
|
assert lot.lon == pytest.approx(60.49186)
|
|
assert lot.address == "Ekaterinburg, ulitsa Evgenia Savkova, 8"
|
|
assert lot.house_source == "yandex_realty_nb"
|
|
assert lot.house_ext_id == "280938"
|
|
assert lot.listing_segment == "vtorichka"
|
|
assert len(lot.photo_urls) == 2
|
|
assert all(u.startswith("https://avatars.mds.yandex.net") for u in lot.photo_urls)
|
|
assert lot.raw_payload is not None
|
|
assert lot.raw_payload["ceiling_height"] == pytest.approx(2.7)
|
|
assert lot.raw_payload["kitchen_area_m2"] == pytest.approx(12.7)
|
|
|
|
|
|
def test_entity_to_lot_segment_vtorichka_default():
|
|
"""Default new_flat='NO' -> listing_segment='vtorichka'."""
|
|
lot = _entity_to_lot(_ENTITY_FULL)
|
|
assert lot is not None
|
|
assert lot.listing_segment == "vtorichka"
|
|
|
|
|
|
def test_entity_to_lot_segment_novostroyki_on_new_flat_yes():
|
|
"""new_flat='YES' -> listing_segment='novostroyki'."""
|
|
lot = _entity_to_lot(_ENTITY_FULL, new_flat="YES")
|
|
assert lot is not None
|
|
assert lot.listing_segment == "novostroyki"
|
|
|
|
|
|
def test_parse_gate_json_propagates_segment():
|
|
"""_parse_gate_json threads new_flat down to each lot's listing_segment."""
|
|
payload = _make_gate_payload([_ENTITY_FULL, _ENTITY_STUDIO])
|
|
lots_nb = _parse_gate_json(payload, new_flat="YES")
|
|
assert lots_nb and all(lot.listing_segment == "novostroyki" for lot in lots_nb)
|
|
lots_vt = _parse_gate_json(payload, new_flat="NO")
|
|
assert lots_vt and all(lot.listing_segment == "vtorichka" for lot in lots_vt)
|
|
|
|
|
|
def test_entity_to_lot_studio_rooms_zero():
|
|
lot = _entity_to_lot(_ENTITY_STUDIO)
|
|
assert lot is not None
|
|
assert lot.rooms == 0
|
|
assert lot.area_m2 == pytest.approx(28.0)
|
|
assert lot.floor == 2
|
|
assert lot.total_floors == 9
|
|
assert lot.house_source is None
|
|
assert lot.house_ext_id is None
|
|
assert lot.photo_urls == []
|
|
|
|
|
|
def test_entity_to_lot_no_price_returns_none():
|
|
lot = _entity_to_lot(_ENTITY_NO_PRICE)
|
|
assert lot is None
|
|
|
|
|
|
def test_entity_to_lot_no_offer_id_returns_none():
|
|
lot = _entity_to_lot(_ENTITY_NO_OFFER_ID)
|
|
assert lot is None
|
|
|
|
|
|
def test_entity_to_lot_photo_url_prefix():
|
|
"""mainImages starting with // must get https: prepended."""
|
|
lot = _entity_to_lot(_ENTITY_FULL)
|
|
assert lot is not None
|
|
# Both URLs must start with https:// (not protocol-relative //)
|
|
assert all(u.startswith("https://") for u in lot.photo_urls)
|
|
assert not any(u.startswith("//") for u in lot.photo_urls)
|
|
|
|
|
|
def test_entity_to_lot_photo_capped_at_5():
|
|
entity = dict(_ENTITY_FULL)
|
|
entity["mainImages"] = [f"//avatars.mds.yandex.net/img{i}/main" for i in range(10)]
|
|
lot = _entity_to_lot(entity)
|
|
assert lot is not None
|
|
assert len(lot.photo_urls) == 5
|
|
|
|
|
|
def test_entity_to_lot_url_fallback_when_missing():
|
|
"""When url field absent, fall back to constructed /offer/<id>/ URL."""
|
|
entity = dict(_ENTITY_FULL)
|
|
entity["url"] = None
|
|
lot = _entity_to_lot(entity)
|
|
assert lot is not None
|
|
expected_id = _ENTITY_FULL["offerId"]
|
|
assert lot.source_url == f"https://realty.yandex.ru/offer/{expected_id}/"
|
|
|
|
|
|
def test_entity_to_lot_no_house_when_site_id_absent():
|
|
"""Entity without siteId produces house_source=None, house_ext_id=None."""
|
|
entity = dict(_ENTITY_FULL)
|
|
entity["building"] = {"builtYear": 2015, "buildingType": "BRICK", "siteId": None}
|
|
lot = _entity_to_lot(entity)
|
|
assert lot is not None
|
|
assert lot.house_source is None
|
|
assert lot.house_ext_id is None
|
|
|
|
|
|
def test_entity_to_lot_page_param_in_raw():
|
|
"""page_param passed to _entity_to_lot appears in raw_payload."""
|
|
lot = _entity_to_lot(_ENTITY_FULL, page_param=3)
|
|
assert lot is not None
|
|
assert lot.raw_payload["page_param"] == 3
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# PART C: _parse_gate_json (list-level)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_parse_gate_json_returns_list():
|
|
payload = _make_gate_payload([_ENTITY_FULL, _ENTITY_STUDIO])
|
|
lots = _parse_gate_json(payload)
|
|
assert len(lots) == 2
|
|
assert all(isinstance(lot, ScrapedLot) for lot in lots)
|
|
|
|
|
|
def test_parse_gate_json_skips_invalid():
|
|
"""Entities that fail validation (no price, no id) are silently skipped."""
|
|
payload = _make_gate_payload([_ENTITY_FULL, _ENTITY_NO_PRICE, _ENTITY_NO_OFFER_ID])
|
|
lots = _parse_gate_json(payload)
|
|
assert len(lots) == 1
|
|
assert lots[0].source_id == str(_ENTITY_FULL["offerId"])
|
|
|
|
|
|
def test_parse_gate_json_empty_entities():
|
|
payload = _make_gate_payload([])
|
|
lots = _parse_gate_json(payload)
|
|
assert lots == []
|
|
|
|
|
|
def test_parse_gate_json_bad_payload():
|
|
"""Error/missing-path payload returns empty list without raising."""
|
|
assert _parse_gate_json({}) == []
|
|
assert _parse_gate_json({"error": "bad"}) == []
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# PART D: _build_url gate-API URL shape
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_build_url_contains_required_params():
|
|
s = YandexRealtyScraper()
|
|
url = s._build_url(page=1)
|
|
assert "realty.yandex.ru/gate/react-page/get/" in url
|
|
assert "rgid=559132" in url
|
|
assert "type=SELL" in url
|
|
assert "category=APARTMENT" in url
|
|
assert "newFlat=NO" in url
|
|
assert "_pageType=search" in url
|
|
assert "_providers=react-search-results-data" in url
|
|
assert "page=1" in url
|
|
|
|
|
|
def test_build_url_page_1_based():
|
|
"""page param is 1-based; page=1 is first page."""
|
|
s = YandexRealtyScraper()
|
|
url1 = s._build_url(page=1)
|
|
url2 = s._build_url(page=2)
|
|
assert "page=1" in url1
|
|
assert "page=2" in url2
|
|
|
|
|
|
def test_build_url_rooms_studio():
|
|
s = YandexRealtyScraper()
|
|
url = s._build_url(page=1, rooms="studio")
|
|
assert "roomsTotal=STUDIO" in url
|
|
|
|
|
|
def test_build_url_rooms_1():
|
|
s = YandexRealtyScraper()
|
|
url = s._build_url(page=1, rooms="1")
|
|
assert "roomsTotal=1" in url
|
|
|
|
|
|
def test_build_url_rooms_4plus():
|
|
s = YandexRealtyScraper()
|
|
url = s._build_url(page=1, rooms="4+")
|
|
assert "roomsTotal=4" in url
|
|
|
|
|
|
def test_build_url_price_range():
|
|
s = YandexRealtyScraper()
|
|
url = s._build_url(page=1, price_min=5_000_000, price_max=7_000_000)
|
|
assert "priceMin=5000000" in url
|
|
assert "priceMax=7000000" in url
|
|
|
|
|
|
def test_build_url_unknown_rooms_no_rooms_param():
|
|
"""Unknown rooms key must not produce a roomsTotal param."""
|
|
s = YandexRealtyScraper()
|
|
url = s._build_url(page=1, rooms="99-room-penthouse")
|
|
assert "roomsTotal" not in url
|
|
|
|
|
|
def test_build_url_combos_are_unique():
|
|
"""All rooms x price-range combos produce distinct URLs."""
|
|
s = YandexRealtyScraper()
|
|
urls = set()
|
|
for rooms in ROOM_PATH.keys():
|
|
for lo, hi in DEFAULT_PRICE_RANGES:
|
|
urls.add(s._build_url(page=1, rooms=rooms, price_min=lo, price_max=hi))
|
|
expected = len(ROOM_PATH) * len(DEFAULT_PRICE_RANGES)
|
|
assert len(urls) == expected, f"URL collisions: {expected} combos but {len(urls)} unique"
|
|
|
|
|
|
def test_build_url_new_flat_default_no():
|
|
"""Default new_flat must keep newFlat=NO (vtorichka) — backward compat."""
|
|
s = YandexRealtyScraper()
|
|
url = s._build_url(page=1)
|
|
assert "newFlat=NO" in url
|
|
assert "newFlat=YES" not in url
|
|
|
|
|
|
def test_build_url_new_flat_yes():
|
|
"""new_flat='YES' must produce newFlat=YES (novostroyki segment)."""
|
|
s = YandexRealtyScraper()
|
|
url = s._build_url(page=1, new_flat="YES")
|
|
assert "newFlat=YES" in url
|
|
|
|
|
|
def test_default_city():
|
|
assert DEFAULT_CITY == "ekaterinburg"
|
|
|
|
|
|
def test_gate_max_pages_cap():
|
|
assert _GATE_MAX_PAGES_CAP == 50
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# PART E: _combo_label
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_combo_label_none_none():
|
|
result = _combo_label(None, 0, 200_000_000)
|
|
assert isinstance(result, str)
|
|
assert "any" in result
|
|
|
|
|
|
def test_combo_label_room_and_price():
|
|
result = _combo_label("1", 5_000_000, 7_000_000)
|
|
assert "1" in result
|
|
assert "5000000" in result
|
|
assert "7000000" in result
|
|
|
|
|
|
def test_combo_label_different_ranges_differ():
|
|
k1 = _combo_label("1", 5_000_000, 7_000_000)
|
|
k2 = _combo_label("1", 7_000_000, 10_000_000)
|
|
k3 = _combo_label("2", 5_000_000, 7_000_000)
|
|
assert k1 != k2
|
|
assert k1 != k3
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# PART F: transport layer
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
# camoufox renders the gate-API JSON wrapped in an HTML <pre> (verified prod 2026-06-17):
|
|
_CAMOUFOX_HTML_PREFIX = (
|
|
'<html><head><link rel="stylesheet" '
|
|
'href="resource://content-accessible/plaintext.css"></head><body><pre>'
|
|
)
|
|
_CAMOUFOX_HTML_SUFFIX = "</pre></body></html>"
|
|
|
|
|
|
def _wrap_camoufox(json_text: str) -> str:
|
|
"""Wrap a JSON string the way camoufox renders a JSON endpoint."""
|
|
return f"{_CAMOUFOX_HTML_PREFIX}{json_text}{_CAMOUFOX_HTML_SUFFIX}"
|
|
|
|
|
|
def test_extract_json_from_content_unwraps_camoufox_pre():
|
|
"""_extract_json_from_content must unwrap camoufox's <pre>{json}</pre> HTML.
|
|
|
|
Feeds the real prod-observed wrapper and asserts the extracted text parses
|
|
back to the original gate-API payload.
|
|
"""
|
|
payload = _make_gate_payload([_ENTITY_FULL])
|
|
wrapped = _wrap_camoufox(json.dumps(payload))
|
|
|
|
extracted = _extract_json_from_content(wrapped)
|
|
assert extracted is not None
|
|
assert extracted.startswith('{"response"')
|
|
assert json.loads(extracted) == payload
|
|
|
|
|
|
def test_extract_json_from_content_raw_json_fallback():
|
|
"""Raw JSON (no HTML wrapper) is returned via the first-'{' fallback."""
|
|
raw = json.dumps(_make_gate_payload([]))
|
|
assert _extract_json_from_content(raw) == raw
|
|
|
|
|
|
def test_extract_json_from_content_returns_none_when_no_json():
|
|
assert _extract_json_from_content("<html><body>nope</body></html>") is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_aexit_closes_browser_and_session():
|
|
s = YandexRealtyScraper()
|
|
mock_browser = AsyncMock()
|
|
mock_session = AsyncMock()
|
|
s._browser = mock_browser
|
|
s._cffi_session = mock_session
|
|
await s.__aexit__(None, None, None)
|
|
mock_browser.__aexit__.assert_awaited_once()
|
|
mock_session.close.assert_awaited_once()
|
|
assert s._browser is None
|
|
assert s._cffi_session is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_http_get_raises_when_no_browser():
|
|
"""_http_get must raise RuntimeError outside async context manager (no browser)."""
|
|
s = YandexRealtyScraper()
|
|
with pytest.raises(RuntimeError, match="async context manager"):
|
|
await s._http_get("https://realty.yandex.ru/test/")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_http_get_unwraps_camoufox_html_to_json_body():
|
|
"""_http_get fetches via BrowserFetcher and returns extracted JSON with status 200."""
|
|
s = YandexRealtyScraper()
|
|
payload = _make_gate_payload([_ENTITY_FULL])
|
|
wrapped = _wrap_camoufox(json.dumps(payload))
|
|
|
|
mock_browser = AsyncMock()
|
|
mock_browser.fetch = AsyncMock(return_value=wrapped)
|
|
s._browser = mock_browser
|
|
|
|
resp = await s._http_get("https://realty.yandex.ru/test/", timeout=30)
|
|
|
|
assert isinstance(resp, _CurlResponse)
|
|
assert resp.status_code == 200
|
|
assert json.loads(resp.text) == payload
|
|
mock_browser.fetch.assert_awaited_once_with("https://realty.yandex.ru/test/")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_http_get_status_0_on_fetch_exception():
|
|
"""BrowserFetcher.fetch raising -> _CurlResponse(status_code=0) (transient failure)."""
|
|
s = YandexRealtyScraper()
|
|
mock_browser = AsyncMock()
|
|
mock_browser.fetch = AsyncMock(side_effect=RuntimeError("browser down"))
|
|
s._browser = mock_browser
|
|
|
|
resp = await s._http_get("https://realty.yandex.ru/test/")
|
|
|
|
assert resp.status_code == 0
|
|
assert resp.text == ""
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_http_get_status_0_when_no_json_in_content():
|
|
"""Content with no JSON blob -> _CurlResponse(status_code=0)."""
|
|
s = YandexRealtyScraper()
|
|
mock_browser = AsyncMock()
|
|
mock_browser.fetch = AsyncMock(return_value="<html><body>blocked</body></html>")
|
|
s._browser = mock_browser
|
|
|
|
resp = await s._http_get("https://realty.yandex.ru/test/")
|
|
|
|
assert resp.status_code == 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# PART G: fetch_around — tarpit resilience
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _valid_gate_text(entity: dict | None = None) -> str:
|
|
ent = entity or _ENTITY_FULL
|
|
pager = {"page": 0, "pageSize": 20, "totalItems": 1, "totalPages": 1}
|
|
return json.dumps(_make_gate_payload([ent], pager))
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_around_returns_lots_on_200_json():
|
|
s = YandexRealtyScraper()
|
|
body = _valid_gate_text()
|
|
|
|
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
|
return _CurlResponse(status_code=200, text=body)
|
|
|
|
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
|
with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
|
|
lots = await s.fetch_around(lat=56.8, lon=60.6, rooms="1")
|
|
|
|
assert len(lots) == 1
|
|
assert lots[0].source_id == str(_ENTITY_FULL["offerId"])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_around_rotates_and_retries_on_status_0():
|
|
"""fetch_around rotates IP and retries on status_code==0 (tarpit)."""
|
|
s = YandexRealtyScraper()
|
|
call_count = 0
|
|
body = _valid_gate_text()
|
|
|
|
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
|
nonlocal call_count
|
|
call_count += 1
|
|
if call_count == 1:
|
|
return _CurlResponse(status_code=0, text="")
|
|
return _CurlResponse(status_code=200, text=body)
|
|
|
|
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
|
with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate:
|
|
with patch("asyncio.sleep", new_callable=AsyncMock):
|
|
with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
|
|
lots = await s.fetch_around(lat=56.84, lon=60.60, rooms="1")
|
|
|
|
assert mock_rotate.call_count == 1
|
|
assert call_count == 2
|
|
assert len(lots) >= 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_around_rotates_on_json_error():
|
|
"""fetch_around rotates when response body is not valid JSON."""
|
|
s = YandexRealtyScraper()
|
|
call_count = 0
|
|
body = _valid_gate_text()
|
|
|
|
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
|
nonlocal call_count
|
|
call_count += 1
|
|
if call_count == 1:
|
|
return _CurlResponse(status_code=200, text="<html>not json</html>")
|
|
return _CurlResponse(status_code=200, text=body)
|
|
|
|
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
|
with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate:
|
|
with patch("asyncio.sleep", new_callable=AsyncMock):
|
|
with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
|
|
lots = await s.fetch_around(lat=56.84, lon=60.60, rooms="2")
|
|
|
|
assert mock_rotate.call_count == 1
|
|
assert len(lots) >= 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_around_exhausts_retries_returns_empty():
|
|
"""fetch_around returns [] when all retries exhausted."""
|
|
s = YandexRealtyScraper()
|
|
|
|
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
|
return _CurlResponse(status_code=0, text="")
|
|
|
|
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
|
with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate:
|
|
with patch("asyncio.sleep", new_callable=AsyncMock):
|
|
lots = await s.fetch_around(lat=56.84, lon=60.60, rooms="1")
|
|
|
|
assert mock_rotate.call_count >= 1
|
|
assert lots == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_around_no_rotate_on_404():
|
|
"""fetch_around does NOT rotate on 404 -- only on tarpit (status==0) / bad JSON."""
|
|
s = YandexRealtyScraper()
|
|
|
|
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
|
return _CurlResponse(status_code=404, text="Not Found")
|
|
|
|
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
|
with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate:
|
|
lots = await s.fetch_around(lat=56.84, lon=60.60)
|
|
|
|
assert mock_rotate.call_count == 0
|
|
assert lots == []
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# PART H: fetch_around_multi_room
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_around_multi_room_dedup():
|
|
"""Multi-room fetch deduplicates offers seen in multiple combos.
|
|
|
|
fetch_around_multi_room fetches page=1 inline via _http_get, not via
|
|
fetch_around. We mock _http_get to return JSON payloads with shared/unique
|
|
offer IDs to verify deduplication.
|
|
"""
|
|
s = YandexRealtyScraper()
|
|
|
|
# Combo 0: shared_offer + unique_0
|
|
# Combo 1: shared_offer + unique_1
|
|
entity_shared = dict(
|
|
_ENTITY_FULL, offerId="shared_offer_1234", url="//realty.yandex.ru/offer/shared_offer_1234"
|
|
)
|
|
entity_unique_0 = dict(
|
|
_ENTITY_FULL, offerId="unique_offer_0", url="//realty.yandex.ru/offer/unique_offer_0"
|
|
)
|
|
entity_unique_1 = dict(
|
|
_ENTITY_FULL, offerId="unique_offer_1", url="//realty.yandex.ru/offer/unique_offer_1"
|
|
)
|
|
|
|
pager = {"page": 0, "pageSize": 20, "totalItems": 2, "totalPages": 1}
|
|
payload_combo0 = _make_gate_payload([entity_shared, entity_unique_0], pager)
|
|
payload_combo1 = _make_gate_payload([entity_shared, entity_unique_1], pager)
|
|
|
|
combo_payloads = [payload_combo0, payload_combo1]
|
|
call_idx = 0
|
|
|
|
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
|
nonlocal call_idx
|
|
idx = call_idx
|
|
call_idx += 1
|
|
if idx < len(combo_payloads):
|
|
return _CurlResponse(status_code=200, text=json.dumps(combo_payloads[idx]))
|
|
return _CurlResponse(status_code=200, text=json.dumps(_make_gate_payload([])))
|
|
|
|
mock_settings = MagicMock()
|
|
mock_settings.yandex_proxy_url = None
|
|
|
|
with patch.dict("sys.modules", {"app.core.config": MagicMock(settings=mock_settings)}):
|
|
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
|
with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
|
|
lots = await s.fetch_around_multi_room(
|
|
lat=0.0,
|
|
lon=0.0,
|
|
rooms_list=["1"],
|
|
price_ranges=[(None, 5_000_000), (5_000_000, 7_000_000)],
|
|
max_pages=1,
|
|
)
|
|
|
|
ids = {lot.source_id for lot in lots}
|
|
assert "shared_offer_1234" in ids
|
|
assert len(ids) == 3
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_around_multi_room_runs_both_segments_with_shared_dedup():
|
|
"""segments=['NO','YES'] sweeps both vtorichka+novostroyki; dedup spans both passes.
|
|
|
|
Mocks _http_get and inspects requested URLs: both newFlat=NO and newFlat=YES
|
|
must be requested. The same offerId returned in both passes must be deduped to
|
|
one lot (single shared `seen` across passes), tagged by the FIRST pass's segment.
|
|
"""
|
|
s = YandexRealtyScraper()
|
|
|
|
requested_new_flat: list[str] = []
|
|
|
|
entity_shared = dict(
|
|
_ENTITY_FULL, offerId="shared_xseg", url="//realty.yandex.ru/offer/shared_xseg"
|
|
)
|
|
entity_vt = dict(_ENTITY_FULL, offerId="vt_only", url="//realty.yandex.ru/offer/vt_only")
|
|
entity_nb = dict(_ENTITY_FULL, offerId="nb_only", url="//realty.yandex.ru/offer/nb_only")
|
|
pager = {"page": 0, "pageSize": 20, "totalItems": 2, "totalPages": 1}
|
|
|
|
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
|
if "newFlat=YES" in url:
|
|
requested_new_flat.append("YES")
|
|
payload = _make_gate_payload([entity_shared, entity_nb], pager)
|
|
else:
|
|
requested_new_flat.append("NO")
|
|
payload = _make_gate_payload([entity_shared, entity_vt], pager)
|
|
return _CurlResponse(status_code=200, text=json.dumps(payload))
|
|
|
|
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
|
with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
|
|
lots = await s.fetch_around_multi_room(
|
|
lat=0.0,
|
|
lon=0.0,
|
|
rooms_list=["1"],
|
|
price_ranges=[(None, 5_000_000)],
|
|
segments=["NO", "YES"],
|
|
max_pages=1,
|
|
)
|
|
|
|
assert "NO" in requested_new_flat
|
|
assert "YES" in requested_new_flat
|
|
|
|
by_id = {lot.source_id: lot for lot in lots}
|
|
assert set(by_id) == {"shared_xseg", "vt_only", "nb_only"}
|
|
# Shared offer deduped to ONE lot, tagged by the first pass (NO -> vtorichka).
|
|
assert by_id["shared_xseg"].listing_segment == "vtorichka"
|
|
assert by_id["vt_only"].listing_segment == "vtorichka"
|
|
assert by_id["nb_only"].listing_segment == "novostroyki"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_around_multi_room_default_segment_no_only():
|
|
"""Without segments arg, only newFlat=NO (vtorichka) is requested (legacy default)."""
|
|
s = YandexRealtyScraper()
|
|
requested_new_flat: list[str] = []
|
|
pager = {"page": 0, "pageSize": 20, "totalItems": 1, "totalPages": 1}
|
|
|
|
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
|
requested_new_flat.append("YES" if "newFlat=YES" in url else "NO")
|
|
body = json.dumps(_make_gate_payload([_ENTITY_FULL], pager))
|
|
return _CurlResponse(status_code=200, text=body)
|
|
|
|
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
|
with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
|
|
await s.fetch_around_multi_room(
|
|
lat=0.0,
|
|
lon=0.0,
|
|
rooms_list=["1"],
|
|
price_ranges=[(None, 5_000_000)],
|
|
max_pages=1,
|
|
)
|
|
|
|
assert set(requested_new_flat) == {"NO"}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_around_multi_room_legacy_single_sweep():
|
|
"""Legacy mode (no rooms_list/price_ranges) uses single citywide sweep."""
|
|
s = YandexRealtyScraper()
|
|
calls: list[dict] = []
|
|
|
|
async def fake_fetch_around(
|
|
lat, lon, radius_m=1000, page=1, rooms=None, price_min=None, price_max=None
|
|
):
|
|
calls.append({"page": page, "rooms": rooms, "price_min": price_min})
|
|
return []
|
|
|
|
with patch.object(s, "fetch_around", side_effect=fake_fetch_around):
|
|
await s.fetch_around_multi_room(lat=0.0, lon=0.0, max_pages=3)
|
|
|
|
assert all(c["rooms"] is None for c in calls)
|
|
assert all(c["price_min"] is None for c in calls)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# PART I: exhaustive load -- fetch_all_secondary + _walk_price_range
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_walk_price_range_splits_when_total_exceeds_cap():
|
|
"""When totalItems > cap, _walk_price_range splits into sub-ranges recursively."""
|
|
s = YandexRealtyScraper()
|
|
|
|
call_ranges: list[tuple] = []
|
|
call_count = 0
|
|
|
|
async def fake_fetch_page_json(rooms, page, price_min, price_max):
|
|
nonlocal call_count
|
|
call_count += 1
|
|
call_ranges.append((price_min, price_max))
|
|
# First call (wide range): 600 items -> triggers split
|
|
total = 600 if call_count == 1 else 10
|
|
entities = [dict(_ENTITY_FULL, offerId=f"offer_{call_count}")]
|
|
pager = {
|
|
"page": 0,
|
|
"pageSize": 20,
|
|
"totalItems": total,
|
|
"totalPages": max(1, total // 20),
|
|
}
|
|
return _make_gate_payload(entities, pager)
|
|
|
|
seen: dict = {}
|
|
with patch.object(s, "_fetch_page_json", side_effect=fake_fetch_page_json):
|
|
with patch("asyncio.sleep", new_callable=AsyncMock):
|
|
await s._walk_price_range(
|
|
rooms="1",
|
|
lo=0,
|
|
hi=10_000_000,
|
|
seen=seen,
|
|
price_cap_per_bucket=500,
|
|
max_pages_per_bucket=50,
|
|
concurrency=2,
|
|
)
|
|
|
|
assert len(call_ranges) >= 3, f"Expected >=3 probes, got {call_ranges}"
|
|
assert call_ranges[0] == (None, 10_000_000) # lo=0 -> _lo_param=None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_walk_price_range_fallback_when_total_none():
|
|
"""totalItems absent in pager triggers paginate-until-empty fallback."""
|
|
s = YandexRealtyScraper()
|
|
call_count = 0
|
|
|
|
async def fake_fetch_page_json(rooms, page, price_min, price_max):
|
|
nonlocal call_count
|
|
call_count += 1
|
|
if call_count == 1:
|
|
# Probe: pager without totalItems
|
|
pager = {"page": 0, "pageSize": 20}
|
|
return _make_gate_payload([_ENTITY_FULL], pager)
|
|
if call_count == 2:
|
|
pager = {"page": 0, "pageSize": 20}
|
|
return _make_gate_payload([dict(_ENTITY_FULL, offerId=f"offer_{call_count}")], pager)
|
|
return _make_gate_payload([])
|
|
|
|
seen: dict = {}
|
|
bucket_calls: list = []
|
|
|
|
with patch.object(s, "_fetch_page_json", side_effect=fake_fetch_page_json):
|
|
with patch("asyncio.sleep", new_callable=AsyncMock):
|
|
await s._walk_price_range(
|
|
rooms="2",
|
|
lo=0,
|
|
hi=15_000_000,
|
|
seen=seen,
|
|
price_cap_per_bucket=500,
|
|
max_pages_per_bucket=10,
|
|
concurrency=2,
|
|
on_bucket=lambda k, n: bucket_calls.append((k, n)),
|
|
)
|
|
|
|
assert len(bucket_calls) >= 1, "on_bucket must fire in fallback path"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_walk_price_range_fires_on_bucket_for_leaf():
|
|
"""on_bucket callback fires once for a leaf bucket with small totalItems."""
|
|
s = YandexRealtyScraper()
|
|
bucket_calls: list = []
|
|
|
|
async def fake_fetch_page_json(rooms, page, price_min, price_max):
|
|
pager = {"page": 0, "pageSize": 20, "totalItems": 5, "totalPages": 1}
|
|
return _make_gate_payload([_ENTITY_FULL], pager)
|
|
|
|
seen: dict = {}
|
|
with patch.object(s, "_fetch_page_json", side_effect=fake_fetch_page_json):
|
|
with patch("asyncio.sleep", new_callable=AsyncMock):
|
|
await s._walk_price_range(
|
|
rooms="1",
|
|
lo=0,
|
|
hi=20_000_000,
|
|
seen=seen,
|
|
price_cap_per_bucket=500,
|
|
max_pages_per_bucket=5,
|
|
concurrency=2,
|
|
on_bucket=lambda k, n: bucket_calls.append(k),
|
|
)
|
|
|
|
assert len(bucket_calls) >= 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_walk_price_range_skip_buckets():
|
|
"""skip_buckets prevents on_bucket from firing for already-done bucket."""
|
|
s = YandexRealtyScraper()
|
|
on_bucket_calls: list = []
|
|
|
|
async def fake_fetch_page_json(rooms, page, price_min, price_max):
|
|
pager = {"page": 0, "pageSize": 20, "totalItems": 10, "totalPages": 1}
|
|
return _make_gate_payload([_ENTITY_FULL], pager)
|
|
|
|
# lo=0 -> _lo_param=None -> bucket key uses None
|
|
skip_set = {_combo_label("1", 0, 10_000_000)}
|
|
seen: dict = {}
|
|
with patch.object(s, "_fetch_page_json", side_effect=fake_fetch_page_json):
|
|
with patch("asyncio.sleep", new_callable=AsyncMock):
|
|
await s._walk_price_range(
|
|
rooms="1",
|
|
lo=0,
|
|
hi=10_000_000,
|
|
seen=seen,
|
|
price_cap_per_bucket=500,
|
|
max_pages_per_bucket=5,
|
|
concurrency=2,
|
|
on_bucket=lambda k, n: on_bucket_calls.append(k),
|
|
skip_buckets=skip_set,
|
|
)
|
|
|
|
assert skip_set.pop() not in on_bucket_calls
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_all_secondary_deduplicates_across_rooms():
|
|
"""fetch_all_secondary deduplicates offers across room buckets."""
|
|
s = YandexRealtyScraper()
|
|
|
|
shared_lot = MagicMock(spec=ScrapedLot)
|
|
shared_lot.source_id = "shared_999"
|
|
shared_lot.source_url = "https://realty.yandex.ru/offer/shared_999/"
|
|
|
|
unique_lot_1 = MagicMock(spec=ScrapedLot)
|
|
unique_lot_1.source_id = "unique_001"
|
|
unique_lot_1.source_url = "https://realty.yandex.ru/offer/unique_001/"
|
|
|
|
unique_lot_2 = MagicMock(spec=ScrapedLot)
|
|
unique_lot_2.source_id = "unique_002"
|
|
unique_lot_2.source_url = "https://realty.yandex.ru/offer/unique_002/"
|
|
|
|
call_count = 0
|
|
|
|
async def fake_walk(*, rooms, lo, hi, seen, **kwargs):
|
|
nonlocal call_count
|
|
seen["shared_999"] = shared_lot
|
|
if call_count == 0:
|
|
seen["unique_001"] = unique_lot_1
|
|
else:
|
|
seen["unique_002"] = unique_lot_2
|
|
call_count += 1
|
|
|
|
with patch.object(s, "_walk_price_range", side_effect=fake_walk):
|
|
result = await s.fetch_all_secondary(rooms_buckets=["1", "2"])
|
|
|
|
ids = {lot.source_id for lot in result}
|
|
assert "shared_999" in ids
|
|
assert "unique_001" in ids
|
|
assert "unique_002" in ids
|
|
assert len(ids) == 3
|