"""Yandex.Nedvizhimost scraper (realty.yandex.ru) -- gate-API JSON parser.
Transport: BrowserFetcher (camoufox real-browser fingerprint via tradein-browser service).
camoufox's fingerprint is not tarpitted by yandex and does not burn the mobile-proxy IP;
raw system-curl over SOCKS5 gets the proxy IP flagged. One BrowserFetcher context is opened
per scraper session (__aenter__) and reused across all page fetches.
camoufox returns the gate-API JSON wrapped in HTML (verified prod 2026-06-17):
...{"response":...}
JSON is extracted via _extract_json_from_content() (handles the wrapper).
API path: GET https://realty.yandex.ru/gate/react-page/get/
_pageType=search&_providers=react-search-results-data
type=SELL&category=APARTMENT&newFlat=NO&rgid=559132 (EKB)
roomsTotal=&priceMin=...&priceMax=...&page=<1-based>
Response shape: response.search.offers.{entities[], pager{page,pageSize,totalItems,totalPages}}
Page param is 1-based: page=1 -> first page (pager.page=0 in response).
page=0 -> API returns error. Paginate from page=1 to totalPages (inclusive).
Studio roomsTotal:
Request: roomsTotal=STUDIO
Response entities: roomsTotal=None -> mapped to rooms=0 (existing convention).
"""
from __future__ import annotations
import asyncio
import json
import logging
import math
import re
from dataclasses import dataclass
from datetime import UTC, date, datetime
from typing import Any
from urllib.parse import urlencode
from curl_cffi.requests import AsyncSession as _CurlCffiSession
from app.services.scraper_settings import get_scraper_delay
from app.services.scrapers.base import BaseScraper, ScrapedLot
from app.services.scrapers.browser_fetcher import BrowserFetcher
from app.services.scrapers.house_type_normalizer import normalize_house_type
from app.services.scrapers.price_brackets import get_price_seed_brackets
logger = logging.getLogger(__name__)
# camoufox renders JSON endpoints as {"response":...} inside an HTML wrapper.
_RE_PRE_JSON = re.compile("]*>(\\{.*?)
", re.DOTALL | re.IGNORECASE)
DEFAULT_CITY = "ekaterinburg"
_GATE_MAX_PAGES_CAP = 50
_EKB_RGID = 559132
_GATE_URL = "https://realty.yandex.ru/gate/react-page/get/"
ROOM_GATE_PARAM: dict[str, str | int] = {
"studio": "STUDIO",
"1": 1,
"2": 2,
"3": 3,
"4+": 4,
}
ROOM_PATH: dict[str, str] = {
"studio": "studiya",
"1": "odnokomnatnaya",
"2": "dvuhkomnatnaya",
"3": "trehkomnatnaya",
"4+": "4-i-bolee",
}
DEFAULT_PRICE_RANGES: list[tuple[int | None, int | None]] = [
(None, 5_000_000),
(5_000_000, 7_000_000),
(7_000_000, 10_000_000),
(10_000_000, 15_000_000),
(15_000_000, 25_000_000),
(25_000_000, None),
]
_YANDEX_MAX_PRICE = 200_000_000
_YANDEX_MIN_BRACKET = 50_000
_GATE_PAGE_SIZE = 20
_YANDEX_TARPIT_MAX_RETRIES: int = 2
# Таймаут httpx-клиента BrowserFetcher для yandex-пути: 30s вместо глобального 120s.
# Один проблемный combo занимает ≤30s×(1+_YANDEX_TARPIT_MAX_RETRIES)=90s вместо 360s.
_YANDEX_BROWSER_FETCH_TIMEOUT_S: float = 30.0
@dataclass
class _CurlResponse:
status_code: int
text: str
def _is_gate_error(payload: dict[str, Any]) -> bool:
"""Return True when gate-API returned an error object instead of search results."""
return "error" in payload or "response" not in payload
def _extract_gate_data(
payload: dict[str, Any],
) -> tuple[list[dict[str, Any]], dict[str, Any]] | None:
"""Extract (entities, pager) from valid gate-API response. None on missing path."""
try:
offers = payload["response"]["search"]["offers"]
entities: list[dict[str, Any]] = offers["entities"]
pager: dict[str, Any] = offers["pager"]
return entities, pager
except (KeyError, TypeError):
return None
def _extract_json_from_content(content: str) -> str | None:
"""Extract raw JSON text from camoufox HTML-wrapped content.
camoufox renders a JSON endpoint as:
...{"response":...}
Strategy:
1. Try ...
regex (primary -- matches prod output).
2. Fallback: slice from the first '{' (handles raw-JSON response, no wrapper).
Returns the JSON string, or None if no '{' is found.
"""
m = _RE_PRE_JSON.search(content)
if m:
return m.group(1)
idx = content.find("{")
if idx != -1:
return content[idx:]
return None
def _segment_for(new_flat: str) -> str:
"""Map a newFlat gate-API value to the listings.listing_segment value."""
return "novostroyki" if new_flat == "YES" else "vtorichka"
def _parse_creation_date(raw: Any) -> tuple[date | None, int | None]:
"""Parse gate-API creationDate ('2026-05-24T15:10:27Z') → (date, days_on_market).
days_on_market = floor((now - creationDate).days), clamped to >= 0.
Returns (None, None) when raw is absent or unparseable (never raises).
"""
if not raw or not isinstance(raw, str):
return None, None
txt = raw.strip()
# datetime.fromisoformat handles 'Z' only from py3.11+, but normalise defensively.
if txt.endswith("Z"):
txt = txt[:-1] + "+00:00"
try:
dt = datetime.fromisoformat(txt)
except ValueError:
return None, None
if dt.tzinfo is None:
dt = dt.replace(tzinfo=UTC)
listing_date = dt.date()
delta_days = (datetime.now(UTC) - dt).days
days_on_market = max(delta_days, 0)
return listing_date, days_on_market
def _author_fields(author: dict[str, Any]) -> tuple[bool | None, bool | None, str | None]:
"""Map entity.author → (is_homeowner, is_pro_seller, agency_name).
author.category ∈ {OWNER, AGENCY, AGENT, ...}.
OWNER → is_homeowner=True
AGENCY / AGENT → is_pro_seller=True, agency_name = organization or agentName
Returns (None, None, None) when author/category absent.
"""
category = (author.get("category") or "").upper()
if not category:
return None, None, None
if category == "OWNER":
return True, None, None
if category in {"AGENCY", "AGENT"}:
agency_name = author.get("organization") or author.get("agentName") or None
return None, True, agency_name
return None, None, None
def _to_bigint(raw: Any) -> int | None:
"""Coerce a gate-API numeric value (str or number) to int. None when unparseable."""
if raw is None:
return None
try:
return int(float(raw))
except (TypeError, ValueError):
return None
def _predicted_prices(entity: dict[str, Any]) -> tuple[int | None, int | None, int | None]:
"""Extract (value, min, max) from entity.predictions.predictedPrice.
Yandex's own per-offer valuation. The gate-API nests it under
``predictions.predictedPrice`` (NOT ``predictedPrice.predictedPrice`` —
that key does not exist; verified live 2026-06-18 against the gate-API).
Fields are strings (e.g. "7554000"). Returns (None, None, None) when the
nested path is absent (e.g. novostroyki offers carry no valuation).
"""
outer = entity.get("predictions") or {}
inner = outer.get("predictedPrice") or {}
if not isinstance(inner, dict):
return None, None, None
return (
_to_bigint(inner.get("value")),
_to_bigint(inner.get("min")),
_to_bigint(inner.get("max")),
)
def _metro_stations(location: dict[str, Any]) -> list[dict[str, Any]]:
"""Build metro_stations [{name, min}] from location.metroList[] (+ location.metro)."""
stations: list[dict[str, Any]] = []
raw_list = location.get("metroList")
if isinstance(raw_list, list):
for m in raw_list:
if isinstance(m, dict) and m.get("name"):
stations.append({"name": m.get("name"), "min": m.get("timeToMetro")})
if not stations:
single = location.get("metro")
if isinstance(single, dict) and single.get("name"):
stations.append({"name": single.get("name"), "min": single.get("timeToMetro")})
return stations
def _parse_gate_json(
payload: dict[str, Any], page_param: int = 1, new_flat: str = "NO"
) -> list[ScrapedLot]:
"""Parse gate-API JSON payload into a list of ScrapedLot.
Verified field mapping (from .issdump/yandex_gate_api_sample.json + live prod 2026-06-17):
offerId -> source_id
url (//realty.yandex.ru/offer/) -> source_url (prepend "https:")
price.value -> price_rub
price.valuePerPart -> price_per_m2
area.value -> area_m2
livingSpace.value -> living_area_m2
kitchenSpace.value -> kitchen_area_m2 (+ raw_payload["kitchen_area_m2"])
roomsTotal (int | None) -> rooms (None -> 0 for studio)
floorsOffered[0] -> floor
floorsTotal -> total_floors
building.builtYear -> year_built
building.buildingType -> house_type (normalize_house_type: MONOLIT->monolith)
ceilingHeight -> ceiling_height_m (+ raw_payload["ceiling_height"])
location.point.latitude/longitude -> lat / lon
location.geocoderAddress -> address (full, with house number)
mainImages[] -> photo_urls (prepend "https:", up to 5)
building.siteId -> house_ext_id (JK linkage)
listing_segment -> "novostroyki" (newFlat=YES) | "vtorichka" (newFlat=NO)
creationDate (ISO Z) -> listing_date / publish_date + days_on_market
description -> description
author.category/organization -> is_homeowner / is_pro_seller / agency_name
predictedPrice.predictedPrice.* -> predicted_price_rub / _min / _max
price.trend / price.previous -> price_trend / price_previous_rub
location.metroList[] -> metro_stations [{name, min}]
offerId -> yandex_offer_id
"""
result = _extract_gate_data(payload)
if result is None:
logger.warning("yandex gate: response missing response.search.offers path")
return []
entities, _pager = result
lots: list[ScrapedLot] = []
for entity in entities:
lot = _entity_to_lot(entity, page_param=page_param, new_flat=new_flat)
if lot is not None:
lots.append(lot)
return lots
def _entity_to_lot(
entity: dict[str, Any], page_param: int = 1, new_flat: str = "NO"
) -> ScrapedLot | None:
"""Convert one gate-API entity dict to ScrapedLot. None on missing required fields."""
try:
offer_id = str(entity.get("offerId") or "")
if not offer_id:
return None
url_raw = entity.get("url") or ""
source_url = f"https:{url_raw}" if url_raw.startswith("//") else url_raw
if not source_url:
source_url = f"https://realty.yandex.ru/offer/{offer_id}/"
price_dict = entity.get("price") or {}
price_val = price_dict.get("value")
if not price_val or price_val <= 0:
return None
price_rub = int(price_val)
price_per_m2_raw = price_dict.get("valuePerPart")
price_per_m2 = int(price_per_m2_raw) if price_per_m2_raw else None
area_dict = entity.get("area") or {}
area_m2_raw = area_dict.get("value")
area_m2 = float(area_m2_raw) if area_m2_raw is not None else None
living_dict = entity.get("livingSpace") or {}
living_area_raw = living_dict.get("value")
living_area_m2 = float(living_area_raw) if living_area_raw is not None else None
kitchen_dict = entity.get("kitchenSpace") or {}
kitchen_area_raw = kitchen_dict.get("value")
kitchen_area_m2 = float(kitchen_area_raw) if kitchen_area_raw is not None else None
rooms_raw = entity.get("roomsTotal")
if rooms_raw is None:
rooms = 0 # studio
else:
try:
rooms = int(rooms_raw)
except (TypeError, ValueError):
rooms = None
floors_offered = entity.get("floorsOffered") or []
floor = int(floors_offered[0]) if floors_offered else None
floors_total_raw = entity.get("floorsTotal")
total_floors = int(floors_total_raw) if floors_total_raw is not None else None
ceiling_height = entity.get("ceilingHeight")
# Колонка ceiling_height — numeric(3,2), max 9.99. Yandex SERP отдаёт мусор
# (видели 18 м) → запись out-of-range уронила бы весь батч DataError'ом
# (per-lot SAVEPOINT ловит только IntegrityError). Берём только правдоподобный
# диапазон 2.0–6.0 м, иначе None; сырое значение остаётся в raw_payload. (#2007)
_ceiling_raw = float(ceiling_height) if ceiling_height is not None else None
ceiling_height_m = (
_ceiling_raw if _ceiling_raw is not None and 2.0 <= _ceiling_raw <= 6.0 else None
)
building = entity.get("building") or {}
year_built_raw = building.get("builtYear")
year_built = int(year_built_raw) if year_built_raw is not None else None
# buildingType — SCREAMING (MONOLIT/BRICK/...). Нормализуем в канон, иначе
# estimator soft-penalty штрафует yandex-аналоги (#2007).
raw_building_type = building.get("buildingType")
house_type = normalize_house_type(raw_building_type)
site_id = building.get("siteId")
site_name = building.get("siteName")
house_source: str | None = None
house_ext_id: str | None = None
if site_id:
house_source = "yandex_realty_nb"
house_ext_id = str(site_id)
location = entity.get("location") or {}
point = location.get("point") or {}
lat_raw = point.get("latitude")
lon_raw = point.get("longitude")
lat = float(lat_raw) if lat_raw is not None else None
lon = float(lon_raw) if lon_raw is not None else None
address = location.get("geocoderAddress") or location.get("streetAddress") or None
# ── Rich gate-API fields ───────────────────────────────────────────
listing_date, days_on_market = _parse_creation_date(entity.get("creationDate"))
description = entity.get("description") or None
author = entity.get("author") or {}
is_homeowner, is_pro_seller, agency_name = _author_fields(author)
predicted_value, predicted_min, predicted_max = _predicted_prices(entity)
price_trend = price_dict.get("trend") or None
price_previous_rub = _to_bigint(price_dict.get("previous"))
metro_stations = _metro_stations(location)
photo_urls: list[str] = []
for img in entity.get("mainImages") or []:
if img:
photo_urls.append(f"https:{img}" if img.startswith("//") else img)
if len(photo_urls) >= 5:
break
return ScrapedLot(
source="yandex",
source_url=source_url,
source_id=offer_id,
address=address,
lat=lat,
lon=lon,
rooms=rooms,
area_m2=area_m2,
living_area_m2=living_area_m2,
floor=floor,
total_floors=total_floors,
year_built=year_built,
house_type=house_type,
kitchen_area_m2=kitchen_area_m2,
ceiling_height_m=ceiling_height_m,
price_rub=price_rub,
price_per_m2=price_per_m2,
photo_urls=photo_urls,
house_source=house_source,
house_ext_id=house_ext_id,
house_url=None,
listing_segment=_segment_for(new_flat),
listing_date=listing_date,
publish_date=listing_date,
days_on_market=days_on_market,
description=description,
is_homeowner=is_homeowner,
is_pro_seller=is_pro_seller,
agency_name=agency_name,
metro_stations=metro_stations,
yandex_offer_id=offer_id,
predicted_price_rub=predicted_value,
predicted_price_min=predicted_min,
predicted_price_max=predicted_max,
price_trend=price_trend,
price_previous_rub=price_previous_rub,
raw_payload={
"page_param": page_param,
"ceiling_height": ceiling_height,
"kitchen_area_m2": kitchen_area_raw,
"raw_building_type": raw_building_type, # сырой SCREAMING — для отладки
"site_name": site_name,
"offer_id": offer_id,
},
)
except Exception:
logger.exception("yandex gate: entity parse failed offer_id=%s", entity.get("offerId"))
return None
class YandexRealtyScraper(BaseScraper):
name = "yandex"
base_url = "https://realty.yandex.ru"
request_delay_sec = 5.0
def __init__(self, city: str = DEFAULT_CITY) -> None:
super().__init__()
self.city = city
self.request_delay_sec = get_scraper_delay(self.name)
self._browser: BrowserFetcher | None = None
# _cffi_session retained only for _rotate_ip (changeip call).
self._cffi_session: _CurlCffiSession | None = None
self._cookies: dict[str, str] = {}
async def __aenter__(self) -> YandexRealtyScraper: # type: ignore[override]
"""Open ONE BrowserFetcher (camoufox) session, reused across all page fetches.
camoufox's real-browser fingerprint avoids the mobile-proxy IP tarpit
escalation that raw system-curl over SOCKS5 triggers. Opening per-page is
expensive and unnecessary.
fetch_timeout_s=_YANDEX_BROWSER_FETCH_TIMEOUT_S (30s) ограничивает потери
на один тайм-аутовый запрос: worst-case 30s×(1+retries)=90s на combo вместо 360s.
"""
self._browser = BrowserFetcher(
source="yandex", fetch_timeout_s=_YANDEX_BROWSER_FETCH_TIMEOUT_S
)
await self._browser.__aenter__()
return self
async def __aexit__(self, *args: object) -> None: # type: ignore[override]
if self._browser is not None:
await self._browser.__aexit__(*args)
self._browser = None
if self._cffi_session is not None:
await self._cffi_session.close()
self._cffi_session = None
async def _http_get(self, url: str, **kwargs: object) -> _CurlResponse: # type: ignore[override]
"""Fetch gate-API URL via the shared BrowserFetcher (camoufox).
camoufox returns HTML-wrapped JSON; the JSON is extracted and returned as
the body of a _CurlResponse so existing callers (json.loads(resp.text),
resp.status_code checks) keep working unchanged.
Maps outcomes onto the curl-style status contract:
- JSON extracted -> status_code=200, text=
- fetch raised / no JSON found -> status_code=0 (treated as transient
failure by the caller retry logic)
"""
if self._browser is None:
raise RuntimeError("YandexRealtyScraper must be used as async context manager")
try:
content = await self._browser.fetch(url)
except Exception:
logger.warning("yandex gate: BrowserFetcher.fetch failed url=%s", url, exc_info=True)
return _CurlResponse(status_code=0, text="")
json_text = _extract_json_from_content(content)
if not json_text:
logger.warning(
"yandex gate: could not extract JSON from content url=%s first200=%r",
url,
content[:200],
)
return _CurlResponse(status_code=0, text="")
return _CurlResponse(status_code=200, text=json_text)
async def _rotate_ip(self) -> bool:
"""Rotate mobile proxy IP via changeip URL. Returns True on success."""
from app.core.config import settings as _settings
rotate_url = _settings.yandex_proxy_rotate_url or _settings.avito_proxy_rotate_url
if not rotate_url:
return False
sep = "&" if "?" in rotate_url else "?"
try:
async with _CurlCffiSession(timeout=30) as rot:
await rot.get(f"{rotate_url}{sep}format=json")
await asyncio.sleep(9)
logger.info("yandex proxy: IP rotated via changeip")
return True
except Exception:
logger.warning("yandex proxy: IP rotation failed", exc_info=True)
return False
def _build_url(
self,
page: int = 1,
rooms: str | None = None,
price_min: int | None = None,
price_max: int | None = None,
new_flat: str = "NO",
) -> str:
"""Build gate-API URL. page: 1-based (page=0 -> API error).
new_flat: "NO" -> vtorichka (default, preserves legacy behavior),
"YES" -> novostroyki (primary-sale / reassignment segment).
"""
params: dict[str, str | int] = {
"rgid": _EKB_RGID,
"type": "SELL",
"category": "APARTMENT",
"newFlat": new_flat,
"_pageType": "search",
"_providers": "react-search-results-data",
"page": page,
}
if rooms and rooms in ROOM_GATE_PARAM:
params["roomsTotal"] = ROOM_GATE_PARAM[rooms]
if price_min is not None:
params["priceMin"] = price_min
if price_max is not None:
params["priceMax"] = price_max
return f"{_GATE_URL}?{urlencode(params)}"
async def _fetch_page_json(
self,
rooms: str | None,
page: int,
price_min: int | None,
price_max: int | None,
new_flat: str = "NO",
) -> dict[str, Any] | None:
"""Fetch one gate-API page. Returns parsed JSON dict or None on error/invalid."""
url = self._build_url(
page=page, rooms=rooms, price_min=price_min, price_max=price_max, new_flat=new_flat
)
try:
resp = await self._http_get(url, timeout=60)
except Exception:
logger.exception("yandex gate: GET failed url=%s", url)
return None
if resp.status_code != 200: # type: ignore[union-attr]
logger.warning("yandex gate: HTTP %d url=%s", resp.status_code, url) # type: ignore[union-attr]
return None
try:
payload: dict[str, Any] = json.loads(resp.text) # type: ignore[union-attr]
except (json.JSONDecodeError, ValueError):
logger.warning("yandex gate: JSON parse failed url=%s", url)
return None
if _is_gate_error(payload):
logger.warning("yandex gate: error response url=%s keys=%s", url, list(payload.keys()))
return None
return payload
async def fetch_around(
self,
lat: float,
lon: float,
radius_m: int = 1000,
page: int = 1,
rooms: str | None = None,
price_min: int | None = None,
price_max: int | None = None,
new_flat: str = "NO",
) -> list[ScrapedLot]:
"""Fetch ONE page of gate-API results. lat/lon/radius_m ignored (uses rgid).
page: 1-based. Tarpit resilience: status_code==0 or JSON error
-> rotate IP + retry up to _YANDEX_TARPIT_MAX_RETRIES times.
Non-tarpit failures (4xx/5xx) are not retried.
new_flat: "NO" -> vtorichka, "YES" -> novostroyki.
"""
url = self._build_url(
page=page, rooms=rooms, price_min=price_min, price_max=price_max, new_flat=new_flat
)
payload: dict[str, Any] | None = None
for attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES):
try:
response = await self._http_get(url, timeout=60)
except Exception:
logger.exception("yandex gate fetch failed: %s", url)
return []
status = response.status_code # type: ignore[union-attr]
if status == 0:
logger.warning(
"yandex gate: status=0 (tarpit?) rooms=%s page=%d"
" -- rotating IP + retry attempt %d",
rooms,
page,
attempt + 1,
)
await self._rotate_ip()
await asyncio.sleep(2)
continue
if status != 200:
logger.warning(
"yandex gate: HTTP %d rooms=%s page=%d url=%s", status, rooms, page, url
)
return []
try:
payload = json.loads(response.text) # type: ignore[union-attr]
except (json.JSONDecodeError, ValueError):
logger.warning(
"yandex gate: JSON parse failed rooms=%s page=%d -- rotating + retry %d",
rooms,
page,
attempt + 1,
)
await self._rotate_ip()
await asyncio.sleep(2)
continue
if _is_gate_error(payload):
logger.warning(
"yandex gate: error payload rooms=%s page=%d keys=%s -- rotating + retry %d",
rooms,
page,
list(payload.keys()),
attempt + 1,
)
await self._rotate_ip()
await asyncio.sleep(2)
payload = None
continue
break
if payload is None:
logger.warning(
"yandex gate: all retries exhausted rooms=%s page=%d -- returning empty",
rooms,
page,
)
return []
lots = _parse_gate_json(payload, page_param=page, new_flat=new_flat)
logger.info(
"yandex gate page=%d rooms=%s price=[%s-%s] newFlat=%s: %d cards",
page,
rooms or "all",
price_min,
price_max,
new_flat,
len(lots),
)
await self.sleep_between_requests()
return lots
async def fetch_around_multi_room(
self,
lat: float,
lon: float,
radius_m: int = 1000,
max_pages: int = _GATE_MAX_PAGES_CAP,
rooms_list: list[str] | None = None,
price_ranges: list[tuple[int | None, int | None]] | None = None,
segments: list[str] | None = None,
on_combo: Any = None,
**_legacy_kwargs: Any,
) -> list[ScrapedLot]:
"""Fetch via segment x rooms x price-range combos; paginate each combo to totalPages.
Pagination driven by pager.totalPages from first page response.
Each combo iterates page=1 .. min(totalPages, max_pages, _GATE_MAX_PAGES_CAP).
Deduplicates by source_id/source_url across ALL combos AND segments (one `seen`).
Legacy mode (rooms_list=None, price_ranges=None): single citywide sweep.
segments: list of newFlat values, default ["NO"] (vtorichka only). Pass
["NO", "YES"] to sweep both vtorichka and novostroyki in one call.
on_combo: опциональный callback(combo_label: str, new_lots: list[ScrapedLot]) -> None.
Вызывается после каждого combo с новыми (не повторными) лотами из него.
Позволяет инкрементальный save: вызывающий код сохраняет new_lots сразу,
не дожидаясь конца sweep'а. Дубли между combo НЕ передаются повторно.
"""
seen: dict[str, ScrapedLot] = {}
_segments = segments or ["NO"]
combos_skipped = 0
if rooms_list is None and price_ranges is None:
combos: list[tuple[str | None, int | None, int | None]] = [(None, None, None)]
else:
r_list = rooms_list or list(ROOM_PATH.keys())
p_ranges = price_ranges or DEFAULT_PRICE_RANGES
combos = [(r, lo, hi) for r in r_list for lo, hi in p_ranges]
for new_flat in _segments:
_seg = _segment_for(new_flat)
for rooms, price_min, price_max in combos:
combo_label = f"{_seg}/{_combo_label(rooms, price_min, price_max)}"
total_pages: int | None = None
combo_new_lots: list[ScrapedLot] = []
combo_skipped = False
for page in range(1, max_pages + 1):
if page == 1:
payload_p1: dict[str, Any] | None = None
p1_url = self._build_url(
page=1,
rooms=rooms,
price_min=price_min,
price_max=price_max,
new_flat=new_flat,
)
for _attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES):
try:
resp = await self._http_get(p1_url, timeout=60)
except Exception:
logger.exception(
"yandex gate combo fetch failed combo=%s", combo_label
)
break
if resp.status_code == 0: # type: ignore[union-attr]
logger.warning(
"yandex gate: tarpit combo=%s page=1 attempt=%d -- rotating",
combo_label,
_attempt + 1,
)
await self._rotate_ip()
await asyncio.sleep(2)
continue
if resp.status_code != 200: # type: ignore[union-attr]
logger.warning(
"yandex gate: HTTP %d combo=%s page=1",
resp.status_code,
combo_label, # type: ignore[union-attr]
)
break
try:
payload_p1 = json.loads(resp.text) # type: ignore[union-attr]
except (json.JSONDecodeError, ValueError):
logger.warning(
"yandex gate: JSON parse error combo=%s page=1", combo_label
)
payload_p1 = None
await self._rotate_ip()
await asyncio.sleep(2)
continue
if _is_gate_error(payload_p1):
logger.warning(
"yandex gate: error payload combo=%s page=1", combo_label
)
payload_p1 = None
break
break
if payload_p1 is None:
logger.warning(
"yandex gate combo [%s] page=1: all retries exhausted -- skipping",
combo_label,
)
combo_skipped = True
combos_skipped += 1
break
result = _extract_gate_data(payload_p1)
if result is None:
combo_skipped = True
combos_skipped += 1
break
_entities_p1, pager_p1 = result
total_pages = min(
pager_p1.get("totalPages", 1),
max_pages,
_GATE_MAX_PAGES_CAP,
)
lots_p1 = _parse_gate_json(payload_p1, page_param=1, new_flat=new_flat)
if not lots_p1:
logger.debug(
"yandex gate combo [%s] page=1: empty -- stopping", combo_label
)
break
for lot in lots_p1:
key = lot.source_id or lot.source_url
if key and key not in seen:
seen[key] = lot
combo_new_lots.append(lot)
await self.sleep_between_requests()
if total_pages <= 1:
break
continue
if total_pages is not None and page > total_pages:
break
lots = await self.fetch_around(
lat,
lon,
radius_m,
page=page,
rooms=rooms,
price_min=price_min,
price_max=price_max,
new_flat=new_flat,
)
if not lots:
logger.debug(
"yandex gate combo [%s] page=%d: empty -- stopping",
combo_label,
page,
)
break
for lot in lots:
key = lot.source_id or lot.source_url
if key and key not in seen:
seen[key] = lot
combo_new_lots.append(lot)
# Инкрементальный save: вызываем on_combo если есть новые лоты.
# Скипнутые combo (combo_skipped=True) не вызывают on_combo.
if on_combo is not None and combo_new_lots and not combo_skipped:
try:
on_combo(combo_label, combo_new_lots)
except Exception:
logger.exception(
"yandex gate: on_combo callback failed combo=%s", combo_label
)
logger.info(
"yandex gate aggregate: %d unique lots (%d combos x %d segments=%s, skipped=%d)",
len(seen),
len(combos),
len(_segments),
_segments,
combos_skipped,
)
return list(seen.values())
async def fetch_all_secondary(
self,
*,
rooms_buckets: list[str] | None = None,
price_cap_per_bucket: int = 500,
max_pages_per_bucket: int = _GATE_MAX_PAGES_CAP,
concurrency: int = 4,
on_bucket: Any = None,
on_progress: Any = None,
skip_buckets: set[str] | None = None,
) -> list[ScrapedLot]:
"""Exhaustive EKB vtorichka load via KOMNATNOST x TSENA partitioning.
Uses gate-API JSON (totalItems from pager) instead of DOM state extraction.
Стартует с общей data-driven seed-сетки ценовых брекетов
(get_price_seed_brackets() — shared EKB grid, общая с avito/cian) вместо
одной корень-бисекции [0, _YANDEX_MAX_PRICE]. Закрытые seed-брекеты передают
hi = br_hi - 1 (priceMax эксклюзивный сверху → нет пересечений), открытый
(br_hi is None) — hi=None (без priceMax, весь хвост люкса). Дедуп по
source_id в общем seen страхует на стыках.
"""
_buckets = rooms_buckets if rooms_buckets is not None else list(ROOM_PATH.keys())
seen: dict[str, ScrapedLot] = {}
for rooms_key in _buckets:
logger.info(
"yandex gate exhaustive: starting rooms=%s (price_cap=%d concurrency=%d)",
rooms_key,
price_cap_per_bucket,
concurrency,
)
before = len(seen)
for br_lo, br_hi in get_price_seed_brackets():
walk_hi = br_hi - 1 if br_hi is not None else None
await self._walk_price_range(
rooms=rooms_key,
lo=br_lo,
hi=walk_hi,
seen=seen,
price_cap_per_bucket=price_cap_per_bucket,
max_pages_per_bucket=max_pages_per_bucket,
concurrency=concurrency,
on_bucket=on_bucket,
skip_buckets=skip_buckets,
)
room_collected = len(seen) - before
logger.info(
"yandex gate exhaustive: rooms=%s done -- collected %d (total unique=%d)",
rooms_key,
room_collected,
len(seen),
)
if on_progress is not None:
on_progress(len(seen))
logger.info("yandex gate exhaustive: DONE -- total unique=%d lots", len(seen))
return list(seen.values())
async def _walk_price_range(
self,
*,
rooms: str | None,
lo: int,
hi: int | None,
seen: dict[str, ScrapedLot],
price_cap_per_bucket: int,
max_pages_per_bucket: int,
concurrency: int = 4,
on_bucket: Any = None,
skip_buckets: set[str] | None = None,
_depth: int = 0,
) -> None:
"""Recursive binary price-range partitioning.
Probe page=1 -> read totalItems from pager.
Closed bracket (hi is not None): totalItems > cap and bracket >= MIN_BRACKET
-> split; else paginate leaf.
Open bracket (hi is None): top seed-bracket without priceMax ceiling. Cannot
split (no mid) -> paginate leaf directly (lux tail tiny, tail-loss accepted).
Fallback: totalItems=None -> paginate-until-empty (safe degradation).
skip_buckets: resume checkpoint -- skip leaf buckets already done.
"""
_lo_param = lo if lo > 0 else None
_hi_repr = "open" if hi is None else str(hi)
probe_payload = await self._fetch_page_json(rooms, 1, _lo_param, hi)
await asyncio.sleep(self.request_delay_sec)
total: int | None = None
probe_lots: list[ScrapedLot] = []
if probe_payload is None:
logger.warning(
"yandex gate: probe failed rooms=%s [%d, %s] depth=%d -- rotating + retry",
rooms,
lo,
_hi_repr,
_depth,
)
rotated = await self._rotate_ip()
if rotated:
probe_payload = await self._fetch_page_json(rooms, 1, _lo_param, hi)
await asyncio.sleep(self.request_delay_sec)
if probe_payload is not None:
result = _extract_gate_data(probe_payload)
if result is not None:
_, pager = result
total = pager.get("totalItems")
probe_lots = _parse_gate_json(probe_payload, page_param=1)
else:
logger.warning(
"yandex gate: probe data extract failed rooms=%s [%d, %s]", rooms, lo, _hi_repr
)
if total is None:
# Degraded path: paginate-until-empty without knowing total
logger.warning(
"yandex gate: totalItems unknown rooms=%s [%d, %s] -- paginate until empty",
rooms,
lo,
_hi_repr,
)
page = 1
bucket_key = _combo_label(rooms, lo, hi)
if skip_buckets and bucket_key in skip_buckets:
return
pages_fetched = 0
while pages_fetched < max_pages_per_bucket:
payload = await self._fetch_page_json(rooms, page, _lo_param, hi)
await asyncio.sleep(self.request_delay_sec)
if payload is None:
break
lots = _parse_gate_json(payload, page_param=page)
if not lots:
break
for lot in lots:
if lot.source_id and lot.source_id not in seen:
seen[lot.source_id] = lot
page += 1
pages_fetched += 1
if on_bucket is not None:
on_bucket(bucket_key, len(seen))
return
# Split or paginate. ОТКРЫТЫЙ брекет (hi is None) делить нельзя (нет mid) —
# сразу пагинируем leaf напрямую (хвост люкса крошечный, tail-loss accepted).
cap = price_cap_per_bucket
_min_bracket = 500_000
if hi is not None and total > cap and (hi - lo) >= _min_bracket and _depth < 8:
mid = (lo + hi) // 2
await self._walk_price_range(
rooms=rooms,
lo=lo,
hi=mid,
seen=seen,
price_cap_per_bucket=cap,
max_pages_per_bucket=max_pages_per_bucket,
concurrency=concurrency,
on_bucket=on_bucket,
skip_buckets=skip_buckets,
_depth=_depth + 1,
)
await self._walk_price_range(
rooms=rooms,
lo=mid + 1,
hi=hi,
seen=seen,
price_cap_per_bucket=cap,
max_pages_per_bucket=max_pages_per_bucket,
concurrency=concurrency,
on_bucket=on_bucket,
skip_buckets=skip_buckets,
_depth=_depth + 1,
)
return
# Leaf bucket: paginate
bucket_key = _combo_label(rooms, lo, hi)
if skip_buckets and bucket_key in skip_buckets:
logger.debug("yandex gate: skip bucket %s (checkpoint)", bucket_key)
return
total_pages = min(
math.ceil(total / 20),
_GATE_MAX_PAGES_CAP,
max_pages_per_bucket,
)
logger.info("yandex gate: leaf bucket %s total=%d pages=%d", bucket_key, total, total_pages)
# Add probe lots (page 1 already fetched)
for lot in probe_lots:
if lot.source_id and lot.source_id not in seen:
seen[lot.source_id] = lot
if total_pages <= 1:
if on_bucket is not None:
on_bucket(bucket_key, len(seen))
return
# Paginate pages 2..total_pages with concurrency
sem = asyncio.Semaphore(concurrency)
async def _fetch_leaf_page(pg: int) -> list[ScrapedLot]:
async with sem:
payload = await self._fetch_page_json(rooms, pg, _lo_param, hi)
await asyncio.sleep(self.request_delay_sec)
if payload is None:
return []
return _parse_gate_json(payload, page_param=pg)
tasks = [_fetch_leaf_page(pg) for pg in range(2, total_pages + 1)]
results = await asyncio.gather(*tasks)
for page_lots in results:
for lot in page_lots:
if lot.source_id and lot.source_id not in seen:
seen[lot.source_id] = lot
if on_bucket is not None:
on_bucket(bucket_key, len(seen))
def _combo_label(rooms: str | None, lo: int | None, hi: int | None) -> str:
"""Уникальная метка бакета для checkpoint-логики."""
return f"{rooms or 'any'}:{lo}-{hi}"