1294 lines
47 KiB
Python
1294 lines
47 KiB
Python
"""Unit tests for YandexRealtyScraper gate-API JSON parser.
|
||
|
||
Replaced DOM/HTML SERP tests (2026-06-17): new transport uses
|
||
GET realty.yandex.ru/gate/react-page/get/ (JSON gate-API) instead of HTML SERP.
|
||
"""
|
||
|
||
import json
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.base import ScrapedLot
|
||
from app.services.scrapers.yandex_realty import (
|
||
_GATE_MAX_PAGES_CAP,
|
||
DEFAULT_CITY,
|
||
DEFAULT_PRICE_RANGES,
|
||
ROOM_PATH,
|
||
YandexRealtyScraper,
|
||
_combo_label,
|
||
_CurlResponse,
|
||
_entity_to_lot,
|
||
_extract_gate_data,
|
||
_extract_json_from_content,
|
||
_is_gate_error,
|
||
_parse_gate_json,
|
||
)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Minimal gate-API JSON fixtures (derived from real captured sample 2026-06-17)
|
||
# pager shape: {page, pageSize, totalItems, totalPages}
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_ENTITY_FULL: dict = {
|
||
"offerId": 4740475460451078271,
|
||
"url": "//realty.yandex.ru/offer/4740475460451078271",
|
||
"price": {
|
||
"currency": "RUR",
|
||
"value": 4500000,
|
||
"valuePerPart": 119048,
|
||
},
|
||
"area": {"value": 37.8, "unit": "SQUARE_METER"},
|
||
"livingSpace": {"value": 20, "unit": "SQUARE_METER"},
|
||
"kitchenSpace": {"value": 12.7, "unit": "SQUARE_METER"},
|
||
"roomsTotal": 1,
|
||
"floorsOffered": [8],
|
||
"floorsTotal": 25,
|
||
"ceilingHeight": 2.7,
|
||
"building": {
|
||
"builtYear": 2016,
|
||
"buildingType": "MONOLIT",
|
||
"siteId": 280938,
|
||
"siteName": "Peremen",
|
||
},
|
||
"location": {
|
||
"geocoderAddress": "Ekaterinburg, ulitsa Evgenia Savkova, 8",
|
||
"point": {"latitude": 56.79512, "longitude": 60.49186},
|
||
},
|
||
"mainImages": [
|
||
"//avatars.mds.yandex.net/get-realty-offers/11904018/abc/main",
|
||
"//avatars.mds.yandex.net/get-realty-offers/14717586/def/main",
|
||
],
|
||
}
|
||
|
||
_ENTITY_STUDIO: dict = {
|
||
"offerId": 9999999,
|
||
"url": "//realty.yandex.ru/offer/9999999",
|
||
"price": {"value": 3200000, "valuePerPart": 114285},
|
||
"area": {"value": 28.0},
|
||
"livingSpace": None,
|
||
"kitchenSpace": None,
|
||
"roomsTotal": None, # studio -> rooms=0
|
||
"floorsOffered": [2],
|
||
"floorsTotal": 9,
|
||
"ceilingHeight": None,
|
||
"building": {"builtYear": 2010, "buildingType": "PANEL", "siteId": None, "siteName": None},
|
||
"location": {
|
||
"geocoderAddress": "Ekaterinburg, ulitsa Lenina, 5",
|
||
"point": {"latitude": 56.838, "longitude": 60.597},
|
||
},
|
||
"mainImages": [],
|
||
}
|
||
|
||
_ENTITY_NO_PRICE: dict = {
|
||
"offerId": 1111,
|
||
"url": "//realty.yandex.ru/offer/1111",
|
||
"price": {"value": 0},
|
||
"area": {"value": 40.0},
|
||
"roomsTotal": 1,
|
||
"floorsOffered": [3],
|
||
"floorsTotal": 5,
|
||
"building": {},
|
||
"location": {"point": {"latitude": 56.8, "longitude": 60.6}},
|
||
"mainImages": [],
|
||
}
|
||
|
||
_ENTITY_NO_OFFER_ID: dict = {
|
||
"offerId": None,
|
||
"price": {"value": 5000000},
|
||
"area": {"value": 50.0},
|
||
}
|
||
|
||
|
||
# Entity carrying the rich gate-API fields (creationDate, description, author,
|
||
# predictedPrice, price.trend/previous, location.metroList) — verified live 2026-06-17.
|
||
_ENTITY_RICH_OWNER: dict = {
|
||
"offerId": 5500000000000000001,
|
||
"url": "//realty.yandex.ru/offer/5500000000000000001",
|
||
"creationDate": "2026-05-24T15:10:27Z",
|
||
"description": "Продаётся светлая квартира с видом на парк.",
|
||
"author": {"category": "OWNER", "agentName": "Иван", "organization": None},
|
||
"price": {
|
||
"value": 7200000,
|
||
"valuePerPart": 150000,
|
||
"trend": "DECREASED",
|
||
"previous": 7500000,
|
||
},
|
||
# Real gate-API nests valuation under predictions.predictedPrice (verified
|
||
# live 2026-06-18) — NOT predictedPrice.predictedPrice.
|
||
"predictions": {"predictedPrice": {"value": "7554000", "min": "7100000", "max": "8000000"}},
|
||
"area": {"value": 48.0},
|
||
"roomsTotal": 2,
|
||
"floorsOffered": [5],
|
||
"floorsTotal": 12,
|
||
"building": {"builtYear": 2018, "buildingType": "MONOLIT"},
|
||
"location": {
|
||
"geocoderAddress": "Ekaterinburg, ulitsa Mira, 10",
|
||
"point": {"latitude": 56.84, "longitude": 60.61},
|
||
"metroList": [
|
||
{"name": "Ploshchad 1905 goda", "timeToMetro": 7},
|
||
{"name": "Geologicheskaya", "timeToMetro": 12},
|
||
],
|
||
},
|
||
"mainImages": [],
|
||
}
|
||
|
||
_ENTITY_RICH_AGENCY: dict = {
|
||
"offerId": 5500000000000000002,
|
||
"url": "//realty.yandex.ru/offer/5500000000000000002",
|
||
"creationDate": "2026-06-01T09:00:00Z",
|
||
"description": "Агентская продажа.",
|
||
"author": {"category": "AGENCY", "agentName": "Петров", "organization": "Агентство «Диал»"},
|
||
"price": {"value": 9900000, "trend": "UNCHANGED"},
|
||
"area": {"value": 60.0},
|
||
"roomsTotal": 3,
|
||
"floorsOffered": [3],
|
||
"floorsTotal": 9,
|
||
"building": {"builtYear": 2012, "buildingType": "BRICK"},
|
||
"location": {
|
||
"geocoderAddress": "Ekaterinburg, ulitsa Lenina, 1",
|
||
"point": {"latitude": 56.83, "longitude": 60.6},
|
||
"metro": {"name": "Dinamo", "timeToMetro": 4},
|
||
},
|
||
"mainImages": [],
|
||
}
|
||
|
||
|
||
def _make_gate_payload(entities: list, pager: dict | None = None) -> dict:
|
||
"""Wrap entities + pager into gate-API response shape."""
|
||
if pager is None:
|
||
pager = {"page": 0, "pageSize": 20, "totalItems": len(entities), "totalPages": 1}
|
||
return {
|
||
"response": {
|
||
"search": {
|
||
"offers": {
|
||
"entities": entities,
|
||
"pager": pager,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# PART A: pure helper functions
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_is_gate_error_on_error_key():
|
||
assert _is_gate_error({"error": "bad request"}) is True
|
||
|
||
|
||
def test_is_gate_error_on_missing_response():
|
||
assert _is_gate_error({"something": "else"}) is True
|
||
|
||
|
||
def test_is_gate_error_on_valid_payload():
|
||
payload = _make_gate_payload([_ENTITY_FULL])
|
||
assert _is_gate_error(payload) is False
|
||
|
||
|
||
def test_extract_gate_data_returns_entities_and_pager():
|
||
payload = _make_gate_payload([_ENTITY_FULL, _ENTITY_STUDIO])
|
||
result = _extract_gate_data(payload)
|
||
assert result is not None
|
||
entities, pager = result
|
||
assert len(entities) == 2
|
||
assert "totalItems" in pager
|
||
|
||
|
||
def test_extract_gate_data_returns_none_on_bad_shape():
|
||
assert _extract_gate_data({}) is None
|
||
assert _extract_gate_data({"response": {}}) is None
|
||
assert _extract_gate_data({"response": {"search": {}}}) is None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# PART B: _entity_to_lot field mapping
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_entity_to_lot_full_entity():
|
||
lot = _entity_to_lot(_ENTITY_FULL)
|
||
assert lot is not None
|
||
assert lot.source == "yandex"
|
||
assert lot.source_id == "4740475460451078271"
|
||
assert lot.source_url == "https://realty.yandex.ru/offer/4740475460451078271"
|
||
assert lot.price_rub == 4_500_000
|
||
assert lot.price_per_m2 == 119_048
|
||
assert lot.area_m2 == pytest.approx(37.8)
|
||
assert lot.living_area_m2 == pytest.approx(20.0)
|
||
assert lot.rooms == 1
|
||
assert lot.floor == 8
|
||
assert lot.total_floors == 25
|
||
assert lot.year_built == 2016
|
||
assert lot.house_type == "MONOLIT"
|
||
assert lot.lat == pytest.approx(56.79512)
|
||
assert lot.lon == pytest.approx(60.49186)
|
||
assert lot.address == "Ekaterinburg, ulitsa Evgenia Savkova, 8"
|
||
assert lot.house_source == "yandex_realty_nb"
|
||
assert lot.house_ext_id == "280938"
|
||
assert lot.listing_segment == "vtorichka"
|
||
assert len(lot.photo_urls) == 2
|
||
assert all(u.startswith("https://avatars.mds.yandex.net") for u in lot.photo_urls)
|
||
assert lot.raw_payload is not None
|
||
assert lot.raw_payload["ceiling_height"] == pytest.approx(2.7)
|
||
assert lot.raw_payload["kitchen_area_m2"] == pytest.approx(12.7)
|
||
|
||
|
||
def test_entity_to_lot_segment_vtorichka_default():
|
||
"""Default new_flat='NO' -> listing_segment='vtorichka'."""
|
||
lot = _entity_to_lot(_ENTITY_FULL)
|
||
assert lot is not None
|
||
assert lot.listing_segment == "vtorichka"
|
||
|
||
|
||
def test_entity_to_lot_segment_novostroyki_on_new_flat_yes():
|
||
"""new_flat='YES' -> listing_segment='novostroyki'."""
|
||
lot = _entity_to_lot(_ENTITY_FULL, new_flat="YES")
|
||
assert lot is not None
|
||
assert lot.listing_segment == "novostroyki"
|
||
|
||
|
||
def test_parse_gate_json_propagates_segment():
|
||
"""_parse_gate_json threads new_flat down to each lot's listing_segment."""
|
||
payload = _make_gate_payload([_ENTITY_FULL, _ENTITY_STUDIO])
|
||
lots_nb = _parse_gate_json(payload, new_flat="YES")
|
||
assert lots_nb and all(lot.listing_segment == "novostroyki" for lot in lots_nb)
|
||
lots_vt = _parse_gate_json(payload, new_flat="NO")
|
||
assert lots_vt and all(lot.listing_segment == "vtorichka" for lot in lots_vt)
|
||
|
||
|
||
def test_entity_to_lot_studio_rooms_zero():
|
||
lot = _entity_to_lot(_ENTITY_STUDIO)
|
||
assert lot is not None
|
||
assert lot.rooms == 0
|
||
assert lot.area_m2 == pytest.approx(28.0)
|
||
assert lot.floor == 2
|
||
assert lot.total_floors == 9
|
||
assert lot.house_source is None
|
||
assert lot.house_ext_id is None
|
||
assert lot.photo_urls == []
|
||
|
||
|
||
def test_entity_to_lot_no_price_returns_none():
|
||
lot = _entity_to_lot(_ENTITY_NO_PRICE)
|
||
assert lot is None
|
||
|
||
|
||
def test_entity_to_lot_no_offer_id_returns_none():
|
||
lot = _entity_to_lot(_ENTITY_NO_OFFER_ID)
|
||
assert lot is None
|
||
|
||
|
||
def test_entity_to_lot_photo_url_prefix():
|
||
"""mainImages starting with // must get https: prepended."""
|
||
lot = _entity_to_lot(_ENTITY_FULL)
|
||
assert lot is not None
|
||
# Both URLs must start with https:// (not protocol-relative //)
|
||
assert all(u.startswith("https://") for u in lot.photo_urls)
|
||
assert not any(u.startswith("//") for u in lot.photo_urls)
|
||
|
||
|
||
def test_entity_to_lot_photo_capped_at_5():
|
||
entity = dict(_ENTITY_FULL)
|
||
entity["mainImages"] = [f"//avatars.mds.yandex.net/img{i}/main" for i in range(10)]
|
||
lot = _entity_to_lot(entity)
|
||
assert lot is not None
|
||
assert len(lot.photo_urls) == 5
|
||
|
||
|
||
def test_entity_to_lot_url_fallback_when_missing():
|
||
"""When url field absent, fall back to constructed /offer/<id>/ URL."""
|
||
entity = dict(_ENTITY_FULL)
|
||
entity["url"] = None
|
||
lot = _entity_to_lot(entity)
|
||
assert lot is not None
|
||
expected_id = _ENTITY_FULL["offerId"]
|
||
assert lot.source_url == f"https://realty.yandex.ru/offer/{expected_id}/"
|
||
|
||
|
||
def test_entity_to_lot_no_house_when_site_id_absent():
|
||
"""Entity without siteId produces house_source=None, house_ext_id=None."""
|
||
entity = dict(_ENTITY_FULL)
|
||
entity["building"] = {"builtYear": 2015, "buildingType": "BRICK", "siteId": None}
|
||
lot = _entity_to_lot(entity)
|
||
assert lot is not None
|
||
assert lot.house_source is None
|
||
assert lot.house_ext_id is None
|
||
|
||
|
||
def test_entity_to_lot_page_param_in_raw():
|
||
"""page_param passed to _entity_to_lot appears in raw_payload."""
|
||
lot = _entity_to_lot(_ENTITY_FULL, page_param=3)
|
||
assert lot is not None
|
||
assert lot.raw_payload["page_param"] == 3
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# PART B2: rich gate-API fields (predictedPrice, seller, dates, trend, metro)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_entity_to_lot_rich_owner_fields():
|
||
lot = _entity_to_lot(_ENTITY_RICH_OWNER)
|
||
assert lot is not None
|
||
# creationDate → listing_date / publish_date / days_on_market
|
||
assert lot.listing_date is not None
|
||
assert lot.listing_date.isoformat() == "2026-05-24"
|
||
assert lot.publish_date == lot.listing_date
|
||
assert lot.days_on_market is not None
|
||
assert lot.days_on_market >= 0
|
||
# description
|
||
assert lot.description == "Продаётся светлая квартира с видом на парк."
|
||
# author OWNER
|
||
assert lot.is_homeowner is True
|
||
assert lot.is_pro_seller is None
|
||
assert lot.agency_name is None
|
||
# predictedPrice
|
||
assert lot.predicted_price_rub == 7_554_000
|
||
assert lot.predicted_price_min == 7_100_000
|
||
assert lot.predicted_price_max == 8_000_000
|
||
# price trend / previous
|
||
assert lot.price_trend == "DECREASED"
|
||
assert lot.price_previous_rub == 7_500_000
|
||
# metro_stations from metroList
|
||
assert lot.metro_stations == [
|
||
{"name": "Ploshchad 1905 goda", "min": 7},
|
||
{"name": "Geologicheskaya", "min": 12},
|
||
]
|
||
# yandex_offer_id
|
||
assert lot.yandex_offer_id == "5500000000000000001"
|
||
|
||
|
||
def test_entity_to_lot_rich_agency_fields():
|
||
lot = _entity_to_lot(_ENTITY_RICH_AGENCY)
|
||
assert lot is not None
|
||
# AGENCY → pro seller + agency_name from organization
|
||
assert lot.is_pro_seller is True
|
||
assert lot.is_homeowner is None
|
||
assert lot.agency_name == "Агентство «Диал»"
|
||
# price.trend present, no previous → price_previous_rub None
|
||
assert lot.price_trend == "UNCHANGED"
|
||
assert lot.price_previous_rub is None
|
||
# no predictedPrice path → all None
|
||
assert lot.predicted_price_rub is None
|
||
assert lot.predicted_price_min is None
|
||
assert lot.predicted_price_max is None
|
||
# single location.metro fallback
|
||
assert lot.metro_stations == [{"name": "Dinamo", "min": 4}]
|
||
|
||
|
||
def test_entity_to_lot_no_rich_fields_are_none():
|
||
"""Base fixture lacks rich fields → all new fields default to None/[] (no crash)."""
|
||
lot = _entity_to_lot(_ENTITY_FULL)
|
||
assert lot is not None
|
||
assert lot.listing_date is None
|
||
assert lot.days_on_market is None
|
||
assert lot.description is None
|
||
assert lot.is_homeowner is None
|
||
assert lot.is_pro_seller is None
|
||
assert lot.agency_name is None
|
||
assert lot.predicted_price_rub is None
|
||
assert lot.price_trend is None
|
||
assert lot.price_previous_rub is None
|
||
assert lot.metro_stations == []
|
||
# yandex_offer_id is always populated from offerId
|
||
assert lot.yandex_offer_id == "4740475460451078271"
|
||
|
||
|
||
def test_entity_to_lot_bad_creation_date_is_safe():
|
||
"""Unparseable creationDate → listing_date/days_on_market None, no exception."""
|
||
entity = dict(_ENTITY_RICH_OWNER)
|
||
entity["creationDate"] = "not-a-date"
|
||
lot = _entity_to_lot(entity)
|
||
assert lot is not None
|
||
assert lot.listing_date is None
|
||
assert lot.days_on_market is None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# PART C: _parse_gate_json (list-level)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_gate_json_returns_list():
|
||
payload = _make_gate_payload([_ENTITY_FULL, _ENTITY_STUDIO])
|
||
lots = _parse_gate_json(payload)
|
||
assert len(lots) == 2
|
||
assert all(isinstance(lot, ScrapedLot) for lot in lots)
|
||
|
||
|
||
def test_parse_gate_json_skips_invalid():
|
||
"""Entities that fail validation (no price, no id) are silently skipped."""
|
||
payload = _make_gate_payload([_ENTITY_FULL, _ENTITY_NO_PRICE, _ENTITY_NO_OFFER_ID])
|
||
lots = _parse_gate_json(payload)
|
||
assert len(lots) == 1
|
||
assert lots[0].source_id == str(_ENTITY_FULL["offerId"])
|
||
|
||
|
||
def test_parse_gate_json_empty_entities():
|
||
payload = _make_gate_payload([])
|
||
lots = _parse_gate_json(payload)
|
||
assert lots == []
|
||
|
||
|
||
def test_parse_gate_json_bad_payload():
|
||
"""Error/missing-path payload returns empty list without raising."""
|
||
assert _parse_gate_json({}) == []
|
||
assert _parse_gate_json({"error": "bad"}) == []
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# PART D: _build_url gate-API URL shape
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_build_url_contains_required_params():
|
||
s = YandexRealtyScraper()
|
||
url = s._build_url(page=1)
|
||
assert "realty.yandex.ru/gate/react-page/get/" in url
|
||
assert "rgid=559132" in url
|
||
assert "type=SELL" in url
|
||
assert "category=APARTMENT" in url
|
||
assert "newFlat=NO" in url
|
||
assert "_pageType=search" in url
|
||
assert "_providers=react-search-results-data" in url
|
||
assert "page=1" in url
|
||
|
||
|
||
def test_build_url_page_1_based():
|
||
"""page param is 1-based; page=1 is first page."""
|
||
s = YandexRealtyScraper()
|
||
url1 = s._build_url(page=1)
|
||
url2 = s._build_url(page=2)
|
||
assert "page=1" in url1
|
||
assert "page=2" in url2
|
||
|
||
|
||
def test_build_url_rooms_studio():
|
||
s = YandexRealtyScraper()
|
||
url = s._build_url(page=1, rooms="studio")
|
||
assert "roomsTotal=STUDIO" in url
|
||
|
||
|
||
def test_build_url_rooms_1():
|
||
s = YandexRealtyScraper()
|
||
url = s._build_url(page=1, rooms="1")
|
||
assert "roomsTotal=1" in url
|
||
|
||
|
||
def test_build_url_rooms_4plus():
|
||
s = YandexRealtyScraper()
|
||
url = s._build_url(page=1, rooms="4+")
|
||
assert "roomsTotal=4" in url
|
||
|
||
|
||
def test_build_url_price_range():
|
||
s = YandexRealtyScraper()
|
||
url = s._build_url(page=1, price_min=5_000_000, price_max=7_000_000)
|
||
assert "priceMin=5000000" in url
|
||
assert "priceMax=7000000" in url
|
||
|
||
|
||
def test_build_url_unknown_rooms_no_rooms_param():
|
||
"""Unknown rooms key must not produce a roomsTotal param."""
|
||
s = YandexRealtyScraper()
|
||
url = s._build_url(page=1, rooms="99-room-penthouse")
|
||
assert "roomsTotal" not in url
|
||
|
||
|
||
def test_build_url_combos_are_unique():
|
||
"""All rooms x price-range combos produce distinct URLs."""
|
||
s = YandexRealtyScraper()
|
||
urls = set()
|
||
for rooms in ROOM_PATH.keys():
|
||
for lo, hi in DEFAULT_PRICE_RANGES:
|
||
urls.add(s._build_url(page=1, rooms=rooms, price_min=lo, price_max=hi))
|
||
expected = len(ROOM_PATH) * len(DEFAULT_PRICE_RANGES)
|
||
assert len(urls) == expected, f"URL collisions: {expected} combos but {len(urls)} unique"
|
||
|
||
|
||
def test_build_url_new_flat_default_no():
|
||
"""Default new_flat must keep newFlat=NO (vtorichka) — backward compat."""
|
||
s = YandexRealtyScraper()
|
||
url = s._build_url(page=1)
|
||
assert "newFlat=NO" in url
|
||
assert "newFlat=YES" not in url
|
||
|
||
|
||
def test_build_url_new_flat_yes():
|
||
"""new_flat='YES' must produce newFlat=YES (novostroyki segment)."""
|
||
s = YandexRealtyScraper()
|
||
url = s._build_url(page=1, new_flat="YES")
|
||
assert "newFlat=YES" in url
|
||
|
||
|
||
def test_default_city():
|
||
assert DEFAULT_CITY == "ekaterinburg"
|
||
|
||
|
||
def test_gate_max_pages_cap():
|
||
assert _GATE_MAX_PAGES_CAP == 50
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# PART E: _combo_label
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_combo_label_none_none():
|
||
result = _combo_label(None, 0, 200_000_000)
|
||
assert isinstance(result, str)
|
||
assert "any" in result
|
||
|
||
|
||
def test_combo_label_room_and_price():
|
||
result = _combo_label("1", 5_000_000, 7_000_000)
|
||
assert "1" in result
|
||
assert "5000000" in result
|
||
assert "7000000" in result
|
||
|
||
|
||
def test_combo_label_different_ranges_differ():
|
||
k1 = _combo_label("1", 5_000_000, 7_000_000)
|
||
k2 = _combo_label("1", 7_000_000, 10_000_000)
|
||
k3 = _combo_label("2", 5_000_000, 7_000_000)
|
||
assert k1 != k2
|
||
assert k1 != k3
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# PART F: transport layer
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
# camoufox renders the gate-API JSON wrapped in an HTML <pre> (verified prod 2026-06-17):
|
||
_CAMOUFOX_HTML_PREFIX = (
|
||
'<html><head><link rel="stylesheet" '
|
||
'href="resource://content-accessible/plaintext.css"></head><body><pre>'
|
||
)
|
||
_CAMOUFOX_HTML_SUFFIX = "</pre></body></html>"
|
||
|
||
|
||
def _wrap_camoufox(json_text: str) -> str:
|
||
"""Wrap a JSON string the way camoufox renders a JSON endpoint."""
|
||
return f"{_CAMOUFOX_HTML_PREFIX}{json_text}{_CAMOUFOX_HTML_SUFFIX}"
|
||
|
||
|
||
def test_extract_json_from_content_unwraps_camoufox_pre():
|
||
"""_extract_json_from_content must unwrap camoufox's <pre>{json}</pre> HTML.
|
||
|
||
Feeds the real prod-observed wrapper and asserts the extracted text parses
|
||
back to the original gate-API payload.
|
||
"""
|
||
payload = _make_gate_payload([_ENTITY_FULL])
|
||
wrapped = _wrap_camoufox(json.dumps(payload))
|
||
|
||
extracted = _extract_json_from_content(wrapped)
|
||
assert extracted is not None
|
||
assert extracted.startswith('{"response"')
|
||
assert json.loads(extracted) == payload
|
||
|
||
|
||
def test_extract_json_from_content_raw_json_fallback():
|
||
"""Raw JSON (no HTML wrapper) is returned via the first-'{' fallback."""
|
||
raw = json.dumps(_make_gate_payload([]))
|
||
assert _extract_json_from_content(raw) == raw
|
||
|
||
|
||
def test_extract_json_from_content_returns_none_when_no_json():
|
||
assert _extract_json_from_content("<html><body>nope</body></html>") is None
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_aexit_closes_browser_and_session():
|
||
s = YandexRealtyScraper()
|
||
mock_browser = AsyncMock()
|
||
mock_session = AsyncMock()
|
||
s._browser = mock_browser
|
||
s._cffi_session = mock_session
|
||
await s.__aexit__(None, None, None)
|
||
mock_browser.__aexit__.assert_awaited_once()
|
||
mock_session.close.assert_awaited_once()
|
||
assert s._browser is None
|
||
assert s._cffi_session is None
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_http_get_raises_when_no_browser():
|
||
"""_http_get must raise RuntimeError outside async context manager (no browser)."""
|
||
s = YandexRealtyScraper()
|
||
with pytest.raises(RuntimeError, match="async context manager"):
|
||
await s._http_get("https://realty.yandex.ru/test/")
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_http_get_unwraps_camoufox_html_to_json_body():
|
||
"""_http_get fetches via BrowserFetcher and returns extracted JSON with status 200."""
|
||
s = YandexRealtyScraper()
|
||
payload = _make_gate_payload([_ENTITY_FULL])
|
||
wrapped = _wrap_camoufox(json.dumps(payload))
|
||
|
||
mock_browser = AsyncMock()
|
||
mock_browser.fetch = AsyncMock(return_value=wrapped)
|
||
s._browser = mock_browser
|
||
|
||
resp = await s._http_get("https://realty.yandex.ru/test/", timeout=30)
|
||
|
||
assert isinstance(resp, _CurlResponse)
|
||
assert resp.status_code == 200
|
||
assert json.loads(resp.text) == payload
|
||
mock_browser.fetch.assert_awaited_once_with("https://realty.yandex.ru/test/")
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_http_get_status_0_on_fetch_exception():
|
||
"""BrowserFetcher.fetch raising -> _CurlResponse(status_code=0) (transient failure)."""
|
||
s = YandexRealtyScraper()
|
||
mock_browser = AsyncMock()
|
||
mock_browser.fetch = AsyncMock(side_effect=RuntimeError("browser down"))
|
||
s._browser = mock_browser
|
||
|
||
resp = await s._http_get("https://realty.yandex.ru/test/")
|
||
|
||
assert resp.status_code == 0
|
||
assert resp.text == ""
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_http_get_status_0_when_no_json_in_content():
|
||
"""Content with no JSON blob -> _CurlResponse(status_code=0)."""
|
||
s = YandexRealtyScraper()
|
||
mock_browser = AsyncMock()
|
||
mock_browser.fetch = AsyncMock(return_value="<html><body>blocked</body></html>")
|
||
s._browser = mock_browser
|
||
|
||
resp = await s._http_get("https://realty.yandex.ru/test/")
|
||
|
||
assert resp.status_code == 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# PART G: fetch_around — tarpit resilience
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _valid_gate_text(entity: dict | None = None) -> str:
|
||
ent = entity or _ENTITY_FULL
|
||
pager = {"page": 0, "pageSize": 20, "totalItems": 1, "totalPages": 1}
|
||
return json.dumps(_make_gate_payload([ent], pager))
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_returns_lots_on_200_json():
|
||
s = YandexRealtyScraper()
|
||
body = _valid_gate_text()
|
||
|
||
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
||
return _CurlResponse(status_code=200, text=body)
|
||
|
||
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
||
with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
|
||
lots = await s.fetch_around(lat=56.8, lon=60.6, rooms="1")
|
||
|
||
assert len(lots) == 1
|
||
assert lots[0].source_id == str(_ENTITY_FULL["offerId"])
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_rotates_and_retries_on_status_0():
|
||
"""fetch_around rotates IP and retries on status_code==0 (tarpit)."""
|
||
s = YandexRealtyScraper()
|
||
call_count = 0
|
||
body = _valid_gate_text()
|
||
|
||
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
||
nonlocal call_count
|
||
call_count += 1
|
||
if call_count == 1:
|
||
return _CurlResponse(status_code=0, text="")
|
||
return _CurlResponse(status_code=200, text=body)
|
||
|
||
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
||
with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate:
|
||
with patch("asyncio.sleep", new_callable=AsyncMock):
|
||
with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
|
||
lots = await s.fetch_around(lat=56.84, lon=60.60, rooms="1")
|
||
|
||
assert mock_rotate.call_count == 1
|
||
assert call_count == 2
|
||
assert len(lots) >= 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_rotates_on_json_error():
|
||
"""fetch_around rotates when response body is not valid JSON."""
|
||
s = YandexRealtyScraper()
|
||
call_count = 0
|
||
body = _valid_gate_text()
|
||
|
||
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
||
nonlocal call_count
|
||
call_count += 1
|
||
if call_count == 1:
|
||
return _CurlResponse(status_code=200, text="<html>not json</html>")
|
||
return _CurlResponse(status_code=200, text=body)
|
||
|
||
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
||
with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate:
|
||
with patch("asyncio.sleep", new_callable=AsyncMock):
|
||
with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
|
||
lots = await s.fetch_around(lat=56.84, lon=60.60, rooms="2")
|
||
|
||
assert mock_rotate.call_count == 1
|
||
assert len(lots) >= 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_exhausts_retries_returns_empty():
|
||
"""fetch_around returns [] when all retries exhausted."""
|
||
s = YandexRealtyScraper()
|
||
|
||
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
||
return _CurlResponse(status_code=0, text="")
|
||
|
||
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
||
with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate:
|
||
with patch("asyncio.sleep", new_callable=AsyncMock):
|
||
lots = await s.fetch_around(lat=56.84, lon=60.60, rooms="1")
|
||
|
||
assert mock_rotate.call_count >= 1
|
||
assert lots == []
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_no_rotate_on_404():
|
||
"""fetch_around does NOT rotate on 404 -- only on tarpit (status==0) / bad JSON."""
|
||
s = YandexRealtyScraper()
|
||
|
||
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
||
return _CurlResponse(status_code=404, text="Not Found")
|
||
|
||
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
||
with patch.object(s, "_rotate_ip", new_callable=AsyncMock) as mock_rotate:
|
||
lots = await s.fetch_around(lat=56.84, lon=60.60)
|
||
|
||
assert mock_rotate.call_count == 0
|
||
assert lots == []
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# PART H: fetch_around_multi_room
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_multi_room_dedup():
|
||
"""Multi-room fetch deduplicates offers seen in multiple combos.
|
||
|
||
fetch_around_multi_room fetches page=1 inline via _http_get, not via
|
||
fetch_around. We mock _http_get to return JSON payloads with shared/unique
|
||
offer IDs to verify deduplication.
|
||
"""
|
||
s = YandexRealtyScraper()
|
||
|
||
# Combo 0: shared_offer + unique_0
|
||
# Combo 1: shared_offer + unique_1
|
||
entity_shared = dict(
|
||
_ENTITY_FULL, offerId="shared_offer_1234", url="//realty.yandex.ru/offer/shared_offer_1234"
|
||
)
|
||
entity_unique_0 = dict(
|
||
_ENTITY_FULL, offerId="unique_offer_0", url="//realty.yandex.ru/offer/unique_offer_0"
|
||
)
|
||
entity_unique_1 = dict(
|
||
_ENTITY_FULL, offerId="unique_offer_1", url="//realty.yandex.ru/offer/unique_offer_1"
|
||
)
|
||
|
||
pager = {"page": 0, "pageSize": 20, "totalItems": 2, "totalPages": 1}
|
||
payload_combo0 = _make_gate_payload([entity_shared, entity_unique_0], pager)
|
||
payload_combo1 = _make_gate_payload([entity_shared, entity_unique_1], pager)
|
||
|
||
combo_payloads = [payload_combo0, payload_combo1]
|
||
call_idx = 0
|
||
|
||
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
||
nonlocal call_idx
|
||
idx = call_idx
|
||
call_idx += 1
|
||
if idx < len(combo_payloads):
|
||
return _CurlResponse(status_code=200, text=json.dumps(combo_payloads[idx]))
|
||
return _CurlResponse(status_code=200, text=json.dumps(_make_gate_payload([])))
|
||
|
||
mock_settings = MagicMock()
|
||
mock_settings.yandex_proxy_url = None
|
||
|
||
with patch.dict("sys.modules", {"app.core.config": MagicMock(settings=mock_settings)}):
|
||
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
||
with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
|
||
lots = await s.fetch_around_multi_room(
|
||
lat=0.0,
|
||
lon=0.0,
|
||
rooms_list=["1"],
|
||
price_ranges=[(None, 5_000_000), (5_000_000, 7_000_000)],
|
||
max_pages=1,
|
||
)
|
||
|
||
ids = {lot.source_id for lot in lots}
|
||
assert "shared_offer_1234" in ids
|
||
assert len(ids) == 3
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_multi_room_runs_both_segments_with_shared_dedup():
|
||
"""segments=['NO','YES'] sweeps both vtorichka+novostroyki; dedup spans both passes.
|
||
|
||
Mocks _http_get and inspects requested URLs: both newFlat=NO and newFlat=YES
|
||
must be requested. The same offerId returned in both passes must be deduped to
|
||
one lot (single shared `seen` across passes), tagged by the FIRST pass's segment.
|
||
"""
|
||
s = YandexRealtyScraper()
|
||
|
||
requested_new_flat: list[str] = []
|
||
|
||
entity_shared = dict(
|
||
_ENTITY_FULL, offerId="shared_xseg", url="//realty.yandex.ru/offer/shared_xseg"
|
||
)
|
||
entity_vt = dict(_ENTITY_FULL, offerId="vt_only", url="//realty.yandex.ru/offer/vt_only")
|
||
entity_nb = dict(_ENTITY_FULL, offerId="nb_only", url="//realty.yandex.ru/offer/nb_only")
|
||
pager = {"page": 0, "pageSize": 20, "totalItems": 2, "totalPages": 1}
|
||
|
||
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
||
if "newFlat=YES" in url:
|
||
requested_new_flat.append("YES")
|
||
payload = _make_gate_payload([entity_shared, entity_nb], pager)
|
||
else:
|
||
requested_new_flat.append("NO")
|
||
payload = _make_gate_payload([entity_shared, entity_vt], pager)
|
||
return _CurlResponse(status_code=200, text=json.dumps(payload))
|
||
|
||
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
||
with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
|
||
lots = await s.fetch_around_multi_room(
|
||
lat=0.0,
|
||
lon=0.0,
|
||
rooms_list=["1"],
|
||
price_ranges=[(None, 5_000_000)],
|
||
segments=["NO", "YES"],
|
||
max_pages=1,
|
||
)
|
||
|
||
assert "NO" in requested_new_flat
|
||
assert "YES" in requested_new_flat
|
||
|
||
by_id = {lot.source_id: lot for lot in lots}
|
||
assert set(by_id) == {"shared_xseg", "vt_only", "nb_only"}
|
||
# Shared offer deduped to ONE lot, tagged by the first pass (NO -> vtorichka).
|
||
assert by_id["shared_xseg"].listing_segment == "vtorichka"
|
||
assert by_id["vt_only"].listing_segment == "vtorichka"
|
||
assert by_id["nb_only"].listing_segment == "novostroyki"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_multi_room_default_segment_no_only():
|
||
"""Without segments arg, only newFlat=NO (vtorichka) is requested (legacy default)."""
|
||
s = YandexRealtyScraper()
|
||
requested_new_flat: list[str] = []
|
||
pager = {"page": 0, "pageSize": 20, "totalItems": 1, "totalPages": 1}
|
||
|
||
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
||
requested_new_flat.append("YES" if "newFlat=YES" in url else "NO")
|
||
body = json.dumps(_make_gate_payload([_ENTITY_FULL], pager))
|
||
return _CurlResponse(status_code=200, text=body)
|
||
|
||
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
||
with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
|
||
await s.fetch_around_multi_room(
|
||
lat=0.0,
|
||
lon=0.0,
|
||
rooms_list=["1"],
|
||
price_ranges=[(None, 5_000_000)],
|
||
max_pages=1,
|
||
)
|
||
|
||
assert set(requested_new_flat) == {"NO"}
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_multi_room_legacy_single_sweep():
|
||
"""Legacy mode (no rooms_list/price_ranges) uses single citywide sweep."""
|
||
s = YandexRealtyScraper()
|
||
calls: list[dict] = []
|
||
|
||
async def fake_fetch_around(
|
||
lat, lon, radius_m=1000, page=1, rooms=None, price_min=None, price_max=None
|
||
):
|
||
calls.append({"page": page, "rooms": rooms, "price_min": price_min})
|
||
return []
|
||
|
||
with patch.object(s, "fetch_around", side_effect=fake_fetch_around):
|
||
await s.fetch_around_multi_room(lat=0.0, lon=0.0, max_pages=3)
|
||
|
||
assert all(c["rooms"] is None for c in calls)
|
||
assert all(c["price_min"] is None for c in calls)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# PART I: exhaustive load -- fetch_all_secondary + _walk_price_range
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_walk_price_range_splits_when_total_exceeds_cap():
|
||
"""When totalItems > cap, _walk_price_range splits into sub-ranges recursively."""
|
||
s = YandexRealtyScraper()
|
||
|
||
call_ranges: list[tuple] = []
|
||
call_count = 0
|
||
|
||
async def fake_fetch_page_json(rooms, page, price_min, price_max):
|
||
nonlocal call_count
|
||
call_count += 1
|
||
call_ranges.append((price_min, price_max))
|
||
# First call (wide range): 600 items -> triggers split
|
||
total = 600 if call_count == 1 else 10
|
||
entities = [dict(_ENTITY_FULL, offerId=f"offer_{call_count}")]
|
||
pager = {
|
||
"page": 0,
|
||
"pageSize": 20,
|
||
"totalItems": total,
|
||
"totalPages": max(1, total // 20),
|
||
}
|
||
return _make_gate_payload(entities, pager)
|
||
|
||
seen: dict = {}
|
||
with patch.object(s, "_fetch_page_json", side_effect=fake_fetch_page_json):
|
||
with patch("asyncio.sleep", new_callable=AsyncMock):
|
||
await s._walk_price_range(
|
||
rooms="1",
|
||
lo=0,
|
||
hi=10_000_000,
|
||
seen=seen,
|
||
price_cap_per_bucket=500,
|
||
max_pages_per_bucket=50,
|
||
concurrency=2,
|
||
)
|
||
|
||
assert len(call_ranges) >= 3, f"Expected >=3 probes, got {call_ranges}"
|
||
assert call_ranges[0] == (None, 10_000_000) # lo=0 -> _lo_param=None
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_walk_price_range_fallback_when_total_none():
|
||
"""totalItems absent in pager triggers paginate-until-empty fallback."""
|
||
s = YandexRealtyScraper()
|
||
call_count = 0
|
||
|
||
async def fake_fetch_page_json(rooms, page, price_min, price_max):
|
||
nonlocal call_count
|
||
call_count += 1
|
||
if call_count == 1:
|
||
# Probe: pager without totalItems
|
||
pager = {"page": 0, "pageSize": 20}
|
||
return _make_gate_payload([_ENTITY_FULL], pager)
|
||
if call_count == 2:
|
||
pager = {"page": 0, "pageSize": 20}
|
||
return _make_gate_payload([dict(_ENTITY_FULL, offerId=f"offer_{call_count}")], pager)
|
||
return _make_gate_payload([])
|
||
|
||
seen: dict = {}
|
||
bucket_calls: list = []
|
||
|
||
with patch.object(s, "_fetch_page_json", side_effect=fake_fetch_page_json):
|
||
with patch("asyncio.sleep", new_callable=AsyncMock):
|
||
await s._walk_price_range(
|
||
rooms="2",
|
||
lo=0,
|
||
hi=15_000_000,
|
||
seen=seen,
|
||
price_cap_per_bucket=500,
|
||
max_pages_per_bucket=10,
|
||
concurrency=2,
|
||
on_bucket=lambda k, n: bucket_calls.append((k, n)),
|
||
)
|
||
|
||
assert len(bucket_calls) >= 1, "on_bucket must fire in fallback path"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_walk_price_range_fires_on_bucket_for_leaf():
|
||
"""on_bucket callback fires once for a leaf bucket with small totalItems."""
|
||
s = YandexRealtyScraper()
|
||
bucket_calls: list = []
|
||
|
||
async def fake_fetch_page_json(rooms, page, price_min, price_max):
|
||
pager = {"page": 0, "pageSize": 20, "totalItems": 5, "totalPages": 1}
|
||
return _make_gate_payload([_ENTITY_FULL], pager)
|
||
|
||
seen: dict = {}
|
||
with patch.object(s, "_fetch_page_json", side_effect=fake_fetch_page_json):
|
||
with patch("asyncio.sleep", new_callable=AsyncMock):
|
||
await s._walk_price_range(
|
||
rooms="1",
|
||
lo=0,
|
||
hi=20_000_000,
|
||
seen=seen,
|
||
price_cap_per_bucket=500,
|
||
max_pages_per_bucket=5,
|
||
concurrency=2,
|
||
on_bucket=lambda k, n: bucket_calls.append(k),
|
||
)
|
||
|
||
assert len(bucket_calls) >= 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_walk_price_range_skip_buckets():
|
||
"""skip_buckets prevents on_bucket from firing for already-done bucket."""
|
||
s = YandexRealtyScraper()
|
||
on_bucket_calls: list = []
|
||
|
||
async def fake_fetch_page_json(rooms, page, price_min, price_max):
|
||
pager = {"page": 0, "pageSize": 20, "totalItems": 10, "totalPages": 1}
|
||
return _make_gate_payload([_ENTITY_FULL], pager)
|
||
|
||
# lo=0 -> _lo_param=None -> bucket key uses None
|
||
skip_set = {_combo_label("1", 0, 10_000_000)}
|
||
seen: dict = {}
|
||
with patch.object(s, "_fetch_page_json", side_effect=fake_fetch_page_json):
|
||
with patch("asyncio.sleep", new_callable=AsyncMock):
|
||
await s._walk_price_range(
|
||
rooms="1",
|
||
lo=0,
|
||
hi=10_000_000,
|
||
seen=seen,
|
||
price_cap_per_bucket=500,
|
||
max_pages_per_bucket=5,
|
||
concurrency=2,
|
||
on_bucket=lambda k, n: on_bucket_calls.append(k),
|
||
skip_buckets=skip_set,
|
||
)
|
||
|
||
assert skip_set.pop() not in on_bucket_calls
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_all_secondary_deduplicates_across_rooms():
|
||
"""fetch_all_secondary deduplicates offers across room buckets."""
|
||
s = YandexRealtyScraper()
|
||
|
||
shared_lot = MagicMock(spec=ScrapedLot)
|
||
shared_lot.source_id = "shared_999"
|
||
shared_lot.source_url = "https://realty.yandex.ru/offer/shared_999/"
|
||
|
||
unique_lot_1 = MagicMock(spec=ScrapedLot)
|
||
unique_lot_1.source_id = "unique_001"
|
||
unique_lot_1.source_url = "https://realty.yandex.ru/offer/unique_001/"
|
||
|
||
unique_lot_2 = MagicMock(spec=ScrapedLot)
|
||
unique_lot_2.source_id = "unique_002"
|
||
unique_lot_2.source_url = "https://realty.yandex.ru/offer/unique_002/"
|
||
|
||
call_count = 0
|
||
|
||
async def fake_walk(*, rooms, lo, hi, seen, **kwargs):
|
||
nonlocal call_count
|
||
seen["shared_999"] = shared_lot
|
||
if call_count == 0:
|
||
seen["unique_001"] = unique_lot_1
|
||
else:
|
||
seen["unique_002"] = unique_lot_2
|
||
call_count += 1
|
||
|
||
with patch.object(s, "_walk_price_range", side_effect=fake_walk):
|
||
result = await s.fetch_all_secondary(rooms_buckets=["1", "2"])
|
||
|
||
ids = {lot.source_id for lot in result}
|
||
assert "shared_999" in ids
|
||
assert "unique_001" in ids
|
||
assert "unique_002" in ids
|
||
assert len(ids) == 3
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_all_secondary_walks_shared_seed_grid():
|
||
"""fetch_all_secondary итерирует общую seed-сетку: len(_SEED) walk-вызовов/комнатность.
|
||
|
||
Закрытые брекеты передают hi = br_hi - 1; открытый (последний) — hi=None.
|
||
"""
|
||
from app.services.scrapers.price_brackets import get_price_seed_brackets
|
||
|
||
seed = get_price_seed_brackets()
|
||
s = YandexRealtyScraper()
|
||
walk_args: list[tuple[int, int | None]] = []
|
||
|
||
async def fake_walk(*, rooms, lo, hi, seen, **kwargs):
|
||
walk_args.append((lo, hi))
|
||
|
||
with patch.object(s, "_walk_price_range", side_effect=fake_walk):
|
||
await s.fetch_all_secondary(rooms_buckets=["1"])
|
||
|
||
assert len(walk_args) == len(
|
||
seed
|
||
), f"Ожидалось {len(seed)} walk-вызовов (seed-брекеты), получено {len(walk_args)}"
|
||
# Первый брекет закрыт: hi = seed[0].hi - 1.
|
||
first_lo, first_hi = seed[0]
|
||
assert first_hi is not None
|
||
assert walk_args[0] == (first_lo, first_hi - 1)
|
||
# Последний брекет открыт: hi=None (без priceMax-потолка).
|
||
last_lo, last_hi = seed[-1]
|
||
assert last_hi is None
|
||
assert walk_args[-1] == (last_lo, None)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_walk_price_range_open_bracket_no_split():
|
||
"""Открытый брекет (hi=None) не делится — пагинируется leaf напрямую (one bucket).
|
||
|
||
totalItems > cap, но hi=None → split невозможен → ровно один leaf-бакет.
|
||
"""
|
||
s = YandexRealtyScraper()
|
||
bucket_calls: list = []
|
||
seen_prices: list[int | None] = []
|
||
|
||
async def fake_fetch_page_json(rooms, page, price_min, price_max):
|
||
seen_prices.append(price_max)
|
||
# total > cap, но 1 страница лотов → leaf пагинируется без split
|
||
pager = {"page": 0, "pageSize": 20, "totalItems": 9999, "totalPages": 1}
|
||
return _make_gate_payload([_ENTITY_FULL], pager)
|
||
|
||
seen: dict = {}
|
||
with patch.object(s, "_fetch_page_json", side_effect=fake_fetch_page_json):
|
||
with patch("asyncio.sleep", new_callable=AsyncMock):
|
||
await s._walk_price_range(
|
||
rooms="1",
|
||
lo=250_000_000,
|
||
hi=None,
|
||
seen=seen,
|
||
price_cap_per_bucket=500,
|
||
max_pages_per_bucket=5,
|
||
concurrency=2,
|
||
on_bucket=lambda k, n: bucket_calls.append(k),
|
||
)
|
||
|
||
# Открытый брекет: НЕ делится (один on_bucket), priceMax всегда None (без потолка).
|
||
assert len(bucket_calls) == 1, f"Открытый брекет не должен делиться, got {bucket_calls}"
|
||
assert all(
|
||
pm is None for pm in seen_prices
|
||
), f"Открытый брекет: priceMax всегда None, got {seen_prices}"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# PART J: fetch_around_multi_room on_combo callback (FIX A — incremental save)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_multi_room_on_combo_called_per_combo():
|
||
"""on_combo вызывается после каждого combo с новыми (деduped) лотами.
|
||
|
||
2 combo (rooms_list=['1'], 2 price_ranges): каждый имеет уникальный лот.
|
||
on_combo должен быть вызван дважды — по одному разу на combo.
|
||
"""
|
||
s = YandexRealtyScraper()
|
||
|
||
entity_0 = dict(_ENTITY_FULL, offerId="combo0_lot", url="//realty.yandex.ru/offer/combo0_lot")
|
||
entity_1 = dict(_ENTITY_FULL, offerId="combo1_lot", url="//realty.yandex.ru/offer/combo1_lot")
|
||
pager = {"page": 0, "pageSize": 20, "totalItems": 1, "totalPages": 1}
|
||
|
||
call_idx = 0
|
||
payloads = [
|
||
_make_gate_payload([entity_0], pager),
|
||
_make_gate_payload([entity_1], pager),
|
||
]
|
||
|
||
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
||
nonlocal call_idx
|
||
idx = call_idx
|
||
call_idx += 1
|
||
if idx < len(payloads):
|
||
return _CurlResponse(status_code=200, text=json.dumps(payloads[idx]))
|
||
return _CurlResponse(status_code=200, text=json.dumps(_make_gate_payload([])))
|
||
|
||
on_combo_calls: list[tuple[str, list]] = []
|
||
|
||
def on_combo(label: str, new_lots: list) -> None:
|
||
on_combo_calls.append((label, list(new_lots)))
|
||
|
||
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
||
with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
|
||
lots = await s.fetch_around_multi_room(
|
||
lat=0.0,
|
||
lon=0.0,
|
||
rooms_list=["1"],
|
||
price_ranges=[(None, 5_000_000), (5_000_000, 10_000_000)],
|
||
max_pages=1,
|
||
on_combo=on_combo,
|
||
)
|
||
|
||
# Два уникальных лота, два combo.
|
||
assert len(lots) == 2
|
||
# on_combo вызван ровно дважды — по одному на combo.
|
||
assert len(on_combo_calls) == 2, f"Ожидалось 2 on_combo вызова, получено {len(on_combo_calls)}"
|
||
# Каждый on_combo получил ровно один новый лот.
|
||
for _label, combo_lots in on_combo_calls:
|
||
assert len(combo_lots) == 1
|
||
# Конкретные source_id в правильных batch'ах.
|
||
all_ids = {lot.source_id for _, combo_lots in on_combo_calls for lot in combo_lots}
|
||
assert all_ids == {"combo0_lot", "combo1_lot"}
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_multi_room_on_combo_not_called_for_failed_combo():
|
||
"""on_combo НЕ вызывается для combo, у которого все page-1 retries исчерпаны.
|
||
|
||
FIX B: bounded retry — если status_code=0 на всех попытках page=1,
|
||
combo пропускается. on_combo не должен быть вызван для пропущенного combo.
|
||
Успешный combo (второй) — on_combo должен сработать.
|
||
"""
|
||
s = YandexRealtyScraper()
|
||
|
||
entity_ok = dict(_ENTITY_FULL, offerId="ok_lot", url="//realty.yandex.ru/offer/ok_lot")
|
||
pager = {"page": 0, "pageSize": 20, "totalItems": 1, "totalPages": 1}
|
||
|
||
call_idx = 0
|
||
|
||
async def fake_http_get(url: str, **kwargs: object) -> _CurlResponse:
|
||
nonlocal call_idx
|
||
call_idx += 1
|
||
# Первые (1 + _YANDEX_TARPIT_MAX_RETRIES) запросы — это retries для combo[0] → fail.
|
||
from app.services.scrapers.yandex_realty import _YANDEX_TARPIT_MAX_RETRIES
|
||
|
||
if call_idx <= 1 + _YANDEX_TARPIT_MAX_RETRIES:
|
||
return _CurlResponse(status_code=0, text="")
|
||
# Следующий запрос — combo[1] page=1 → success.
|
||
return _CurlResponse(
|
||
status_code=200, text=json.dumps(_make_gate_payload([entity_ok], pager))
|
||
)
|
||
|
||
on_combo_calls: list[str] = []
|
||
|
||
def on_combo(label: str, new_lots: list) -> None:
|
||
on_combo_calls.append(label)
|
||
|
||
with patch.object(s, "_http_get", side_effect=fake_http_get):
|
||
with patch.object(s, "_rotate_ip", new_callable=AsyncMock):
|
||
with patch("asyncio.sleep", new_callable=AsyncMock):
|
||
with patch.object(s, "sleep_between_requests", new_callable=AsyncMock):
|
||
lots = await s.fetch_around_multi_room(
|
||
lat=0.0,
|
||
lon=0.0,
|
||
rooms_list=["1"],
|
||
price_ranges=[(None, 5_000_000), (5_000_000, 10_000_000)],
|
||
max_pages=1,
|
||
on_combo=on_combo,
|
||
)
|
||
|
||
# combo[0] скипнут, combo[1] успешен → 1 лот в итоге.
|
||
assert len(lots) == 1
|
||
assert lots[0].source_id == "ok_lot"
|
||
# on_combo вызван только для успешного combo, не для скипнутого.
|
||
assert len(on_combo_calls) == 1, (
|
||
f"on_combo должен быть вызван только 1 раз (успешный combo), "
|
||
f"получено {len(on_combo_calls)}: {on_combo_calls}"
|
||
)
|