lekss361
440bb2ca51
All checks were successful
Deploy Trade-In / deploy (push) Successful in 53s
Deploy Trade-In / changes (push) Successful in 11s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 1m30s
Deploy Trade-In / build-backend (push) Successful in 54s
house_type normalized via shared house_type_normalizer (canon = _IMV_HOUSE_TYPE_MAP) — fixes estimator soft-penalty false-mismatch on ~70% yandex analogs; backfill migration 140 (SCREAMING->canon + 'other'->NULL). kitchen/ceiling promoted to columns (dashboard/matching, not valuation — see #2012); ceiling capped 2.0-6.0m to avoid numeric(3,2) overflow. Refs #2007
1085 lines
44 KiB
Python
1085 lines
44 KiB
Python
"""Yandex.Nedvizhimost scraper (realty.yandex.ru) -- gate-API JSON parser.
|
||
|
||
Transport: BrowserFetcher (camoufox real-browser fingerprint via tradein-browser service).
|
||
camoufox's fingerprint is not tarpitted by yandex and does not burn the mobile-proxy IP;
|
||
raw system-curl over SOCKS5 gets the proxy IP flagged. One BrowserFetcher context is opened
|
||
per scraper session (__aenter__) and reused across all page fetches.
|
||
|
||
camoufox returns the gate-API JSON wrapped in HTML (verified prod 2026-06-17):
|
||
<html><head>...</head><body><pre>{"response":...}</pre></body></html>
|
||
JSON is extracted via _extract_json_from_content() (handles the <pre> wrapper).
|
||
|
||
API path: GET https://realty.yandex.ru/gate/react-page/get/
|
||
_pageType=search&_providers=react-search-results-data
|
||
type=SELL&category=APARTMENT&newFlat=NO&rgid=559132 (EKB)
|
||
roomsTotal=<int|STUDIO>&priceMin=...&priceMax=...&page=<1-based>
|
||
|
||
Response shape: response.search.offers.{entities[], pager{page,pageSize,totalItems,totalPages}}
|
||
|
||
Page param is 1-based: page=1 -> first page (pager.page=0 in response).
|
||
page=0 -> API returns error. Paginate from page=1 to totalPages (inclusive).
|
||
|
||
Studio roomsTotal:
|
||
Request: roomsTotal=STUDIO
|
||
Response entities: roomsTotal=None -> mapped to rooms=0 (existing convention).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import logging
|
||
import math
|
||
import re
|
||
from dataclasses import dataclass
|
||
from datetime import UTC, date, datetime
|
||
from typing import Any
|
||
from urllib.parse import urlencode
|
||
|
||
from curl_cffi.requests import AsyncSession as _CurlCffiSession
|
||
|
||
from app.services.scraper_settings import get_scraper_delay
|
||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
from app.services.scrapers.house_type_normalizer import normalize_house_type
|
||
from app.services.scrapers.price_brackets import get_price_seed_brackets
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# camoufox renders JSON endpoints as <pre>{"response":...}</pre> inside an HTML wrapper.
|
||
_RE_PRE_JSON = re.compile("<pre[^>]*>(\\{.*?)</pre>", re.DOTALL | re.IGNORECASE)
|
||
|
||
DEFAULT_CITY = "ekaterinburg"
|
||
_GATE_MAX_PAGES_CAP = 50
|
||
_EKB_RGID = 559132
|
||
_GATE_URL = "https://realty.yandex.ru/gate/react-page/get/"
|
||
|
||
ROOM_GATE_PARAM: dict[str, str | int] = {
|
||
"studio": "STUDIO",
|
||
"1": 1,
|
||
"2": 2,
|
||
"3": 3,
|
||
"4+": 4,
|
||
}
|
||
|
||
ROOM_PATH: dict[str, str] = {
|
||
"studio": "studiya",
|
||
"1": "odnokomnatnaya",
|
||
"2": "dvuhkomnatnaya",
|
||
"3": "trehkomnatnaya",
|
||
"4+": "4-i-bolee",
|
||
}
|
||
|
||
DEFAULT_PRICE_RANGES: list[tuple[int | None, int | None]] = [
|
||
(None, 5_000_000),
|
||
(5_000_000, 7_000_000),
|
||
(7_000_000, 10_000_000),
|
||
(10_000_000, 15_000_000),
|
||
(15_000_000, 25_000_000),
|
||
(25_000_000, None),
|
||
]
|
||
|
||
_YANDEX_MAX_PRICE = 200_000_000
|
||
_YANDEX_MIN_BRACKET = 50_000
|
||
_GATE_PAGE_SIZE = 20
|
||
_YANDEX_TARPIT_MAX_RETRIES: int = 2
|
||
|
||
# Таймаут httpx-клиента BrowserFetcher для yandex-пути: 30s вместо глобального 120s.
|
||
# Один проблемный combo занимает ≤30s×(1+_YANDEX_TARPIT_MAX_RETRIES)=90s вместо 360s.
|
||
_YANDEX_BROWSER_FETCH_TIMEOUT_S: float = 30.0
|
||
|
||
|
||
@dataclass
|
||
class _CurlResponse:
|
||
status_code: int
|
||
text: str
|
||
|
||
|
||
def _is_gate_error(payload: dict[str, Any]) -> bool:
|
||
"""Return True when gate-API returned an error object instead of search results."""
|
||
return "error" in payload or "response" not in payload
|
||
|
||
|
||
def _extract_gate_data(
|
||
payload: dict[str, Any],
|
||
) -> tuple[list[dict[str, Any]], dict[str, Any]] | None:
|
||
"""Extract (entities, pager) from valid gate-API response. None on missing path."""
|
||
try:
|
||
offers = payload["response"]["search"]["offers"]
|
||
entities: list[dict[str, Any]] = offers["entities"]
|
||
pager: dict[str, Any] = offers["pager"]
|
||
return entities, pager
|
||
except (KeyError, TypeError):
|
||
return None
|
||
|
||
|
||
def _extract_json_from_content(content: str) -> str | None:
|
||
"""Extract raw JSON text from camoufox HTML-wrapped content.
|
||
|
||
camoufox renders a JSON endpoint as:
|
||
<html><head>...</head><body><pre>{"response":...}</pre></body></html>
|
||
|
||
Strategy:
|
||
1. Try <pre>...</pre> regex (primary -- matches prod output).
|
||
2. Fallback: slice from the first '{' (handles raw-JSON response, no wrapper).
|
||
|
||
Returns the JSON string, or None if no '{' is found.
|
||
"""
|
||
m = _RE_PRE_JSON.search(content)
|
||
if m:
|
||
return m.group(1)
|
||
|
||
idx = content.find("{")
|
||
if idx != -1:
|
||
return content[idx:]
|
||
|
||
return None
|
||
|
||
|
||
def _segment_for(new_flat: str) -> str:
|
||
"""Map a newFlat gate-API value to the listings.listing_segment value."""
|
||
return "novostroyki" if new_flat == "YES" else "vtorichka"
|
||
|
||
|
||
def _parse_creation_date(raw: Any) -> tuple[date | None, int | None]:
|
||
"""Parse gate-API creationDate ('2026-05-24T15:10:27Z') → (date, days_on_market).
|
||
|
||
days_on_market = floor((now - creationDate).days), clamped to >= 0.
|
||
Returns (None, None) when raw is absent or unparseable (never raises).
|
||
"""
|
||
if not raw or not isinstance(raw, str):
|
||
return None, None
|
||
txt = raw.strip()
|
||
# datetime.fromisoformat handles 'Z' only from py3.11+, but normalise defensively.
|
||
if txt.endswith("Z"):
|
||
txt = txt[:-1] + "+00:00"
|
||
try:
|
||
dt = datetime.fromisoformat(txt)
|
||
except ValueError:
|
||
return None, None
|
||
if dt.tzinfo is None:
|
||
dt = dt.replace(tzinfo=UTC)
|
||
listing_date = dt.date()
|
||
delta_days = (datetime.now(UTC) - dt).days
|
||
days_on_market = max(delta_days, 0)
|
||
return listing_date, days_on_market
|
||
|
||
|
||
def _author_fields(author: dict[str, Any]) -> tuple[bool | None, bool | None, str | None]:
|
||
"""Map entity.author → (is_homeowner, is_pro_seller, agency_name).
|
||
|
||
author.category ∈ {OWNER, AGENCY, AGENT, ...}.
|
||
OWNER → is_homeowner=True
|
||
AGENCY / AGENT → is_pro_seller=True, agency_name = organization or agentName
|
||
Returns (None, None, None) when author/category absent.
|
||
"""
|
||
category = (author.get("category") or "").upper()
|
||
if not category:
|
||
return None, None, None
|
||
if category == "OWNER":
|
||
return True, None, None
|
||
if category in {"AGENCY", "AGENT"}:
|
||
agency_name = author.get("organization") or author.get("agentName") or None
|
||
return None, True, agency_name
|
||
return None, None, None
|
||
|
||
|
||
def _to_bigint(raw: Any) -> int | None:
|
||
"""Coerce a gate-API numeric value (str or number) to int. None when unparseable."""
|
||
if raw is None:
|
||
return None
|
||
try:
|
||
return int(float(raw))
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def _predicted_prices(entity: dict[str, Any]) -> tuple[int | None, int | None, int | None]:
|
||
"""Extract (value, min, max) from entity.predictions.predictedPrice.
|
||
|
||
Yandex's own per-offer valuation. The gate-API nests it under
|
||
``predictions.predictedPrice`` (NOT ``predictedPrice.predictedPrice`` —
|
||
that key does not exist; verified live 2026-06-18 against the gate-API).
|
||
Fields are strings (e.g. "7554000"). Returns (None, None, None) when the
|
||
nested path is absent (e.g. novostroyki offers carry no valuation).
|
||
"""
|
||
outer = entity.get("predictions") or {}
|
||
inner = outer.get("predictedPrice") or {}
|
||
if not isinstance(inner, dict):
|
||
return None, None, None
|
||
return (
|
||
_to_bigint(inner.get("value")),
|
||
_to_bigint(inner.get("min")),
|
||
_to_bigint(inner.get("max")),
|
||
)
|
||
|
||
|
||
def _metro_stations(location: dict[str, Any]) -> list[dict[str, Any]]:
|
||
"""Build metro_stations [{name, min}] from location.metroList[] (+ location.metro)."""
|
||
stations: list[dict[str, Any]] = []
|
||
raw_list = location.get("metroList")
|
||
if isinstance(raw_list, list):
|
||
for m in raw_list:
|
||
if isinstance(m, dict) and m.get("name"):
|
||
stations.append({"name": m.get("name"), "min": m.get("timeToMetro")})
|
||
if not stations:
|
||
single = location.get("metro")
|
||
if isinstance(single, dict) and single.get("name"):
|
||
stations.append({"name": single.get("name"), "min": single.get("timeToMetro")})
|
||
return stations
|
||
|
||
|
||
def _parse_gate_json(
|
||
payload: dict[str, Any], page_param: int = 1, new_flat: str = "NO"
|
||
) -> list[ScrapedLot]:
|
||
"""Parse gate-API JSON payload into a list of ScrapedLot.
|
||
|
||
Verified field mapping (from .issdump/yandex_gate_api_sample.json + live prod 2026-06-17):
|
||
offerId -> source_id
|
||
url (//realty.yandex.ru/offer/) -> source_url (prepend "https:")
|
||
price.value -> price_rub
|
||
price.valuePerPart -> price_per_m2
|
||
area.value -> area_m2
|
||
livingSpace.value -> living_area_m2
|
||
kitchenSpace.value -> kitchen_area_m2 (+ raw_payload["kitchen_area_m2"])
|
||
roomsTotal (int | None) -> rooms (None -> 0 for studio)
|
||
floorsOffered[0] -> floor
|
||
floorsTotal -> total_floors
|
||
building.builtYear -> year_built
|
||
building.buildingType -> house_type (normalize_house_type: MONOLIT->monolith)
|
||
ceilingHeight -> ceiling_height_m (+ raw_payload["ceiling_height"])
|
||
location.point.latitude/longitude -> lat / lon
|
||
location.geocoderAddress -> address (full, with house number)
|
||
mainImages[] -> photo_urls (prepend "https:", up to 5)
|
||
building.siteId -> house_ext_id (JK linkage)
|
||
listing_segment -> "novostroyki" (newFlat=YES) | "vtorichka" (newFlat=NO)
|
||
creationDate (ISO Z) -> listing_date / publish_date + days_on_market
|
||
description -> description
|
||
author.category/organization -> is_homeowner / is_pro_seller / agency_name
|
||
predictedPrice.predictedPrice.* -> predicted_price_rub / _min / _max
|
||
price.trend / price.previous -> price_trend / price_previous_rub
|
||
location.metroList[] -> metro_stations [{name, min}]
|
||
offerId -> yandex_offer_id
|
||
"""
|
||
result = _extract_gate_data(payload)
|
||
if result is None:
|
||
logger.warning("yandex gate: response missing response.search.offers path")
|
||
return []
|
||
|
||
entities, _pager = result
|
||
lots: list[ScrapedLot] = []
|
||
for entity in entities:
|
||
lot = _entity_to_lot(entity, page_param=page_param, new_flat=new_flat)
|
||
if lot is not None:
|
||
lots.append(lot)
|
||
return lots
|
||
|
||
|
||
def _entity_to_lot(
|
||
entity: dict[str, Any], page_param: int = 1, new_flat: str = "NO"
|
||
) -> ScrapedLot | None:
|
||
"""Convert one gate-API entity dict to ScrapedLot. None on missing required fields."""
|
||
try:
|
||
offer_id = str(entity.get("offerId") or "")
|
||
if not offer_id:
|
||
return None
|
||
|
||
url_raw = entity.get("url") or ""
|
||
source_url = f"https:{url_raw}" if url_raw.startswith("//") else url_raw
|
||
if not source_url:
|
||
source_url = f"https://realty.yandex.ru/offer/{offer_id}/"
|
||
|
||
price_dict = entity.get("price") or {}
|
||
price_val = price_dict.get("value")
|
||
if not price_val or price_val <= 0:
|
||
return None
|
||
price_rub = int(price_val)
|
||
|
||
price_per_m2_raw = price_dict.get("valuePerPart")
|
||
price_per_m2 = int(price_per_m2_raw) if price_per_m2_raw else None
|
||
|
||
area_dict = entity.get("area") or {}
|
||
area_m2_raw = area_dict.get("value")
|
||
area_m2 = float(area_m2_raw) if area_m2_raw is not None else None
|
||
|
||
living_dict = entity.get("livingSpace") or {}
|
||
living_area_raw = living_dict.get("value")
|
||
living_area_m2 = float(living_area_raw) if living_area_raw is not None else None
|
||
|
||
kitchen_dict = entity.get("kitchenSpace") or {}
|
||
kitchen_area_raw = kitchen_dict.get("value")
|
||
kitchen_area_m2 = float(kitchen_area_raw) if kitchen_area_raw is not None else None
|
||
|
||
rooms_raw = entity.get("roomsTotal")
|
||
if rooms_raw is None:
|
||
rooms = 0 # studio
|
||
else:
|
||
try:
|
||
rooms = int(rooms_raw)
|
||
except (TypeError, ValueError):
|
||
rooms = None
|
||
|
||
floors_offered = entity.get("floorsOffered") or []
|
||
floor = int(floors_offered[0]) if floors_offered else None
|
||
floors_total_raw = entity.get("floorsTotal")
|
||
total_floors = int(floors_total_raw) if floors_total_raw is not None else None
|
||
|
||
ceiling_height = entity.get("ceilingHeight")
|
||
# Колонка ceiling_height — numeric(3,2), max 9.99. Yandex SERP отдаёт мусор
|
||
# (видели 18 м) → запись out-of-range уронила бы весь батч DataError'ом
|
||
# (per-lot SAVEPOINT ловит только IntegrityError). Берём только правдоподобный
|
||
# диапазон 2.0–6.0 м, иначе None; сырое значение остаётся в raw_payload. (#2007)
|
||
_ceiling_raw = float(ceiling_height) if ceiling_height is not None else None
|
||
ceiling_height_m = (
|
||
_ceiling_raw if _ceiling_raw is not None and 2.0 <= _ceiling_raw <= 6.0 else None
|
||
)
|
||
|
||
building = entity.get("building") or {}
|
||
year_built_raw = building.get("builtYear")
|
||
year_built = int(year_built_raw) if year_built_raw is not None else None
|
||
# buildingType — SCREAMING (MONOLIT/BRICK/...). Нормализуем в канон, иначе
|
||
# estimator soft-penalty штрафует yandex-аналоги (#2007).
|
||
raw_building_type = building.get("buildingType")
|
||
house_type = normalize_house_type(raw_building_type)
|
||
|
||
site_id = building.get("siteId")
|
||
site_name = building.get("siteName")
|
||
house_source: str | None = None
|
||
house_ext_id: str | None = None
|
||
if site_id:
|
||
house_source = "yandex_realty_nb"
|
||
house_ext_id = str(site_id)
|
||
|
||
location = entity.get("location") or {}
|
||
point = location.get("point") or {}
|
||
lat_raw = point.get("latitude")
|
||
lon_raw = point.get("longitude")
|
||
lat = float(lat_raw) if lat_raw is not None else None
|
||
lon = float(lon_raw) if lon_raw is not None else None
|
||
|
||
address = location.get("geocoderAddress") or location.get("streetAddress") or None
|
||
|
||
# ── Rich gate-API fields ───────────────────────────────────────────
|
||
listing_date, days_on_market = _parse_creation_date(entity.get("creationDate"))
|
||
description = entity.get("description") or None
|
||
|
||
author = entity.get("author") or {}
|
||
is_homeowner, is_pro_seller, agency_name = _author_fields(author)
|
||
|
||
predicted_value, predicted_min, predicted_max = _predicted_prices(entity)
|
||
|
||
price_trend = price_dict.get("trend") or None
|
||
price_previous_rub = _to_bigint(price_dict.get("previous"))
|
||
|
||
metro_stations = _metro_stations(location)
|
||
|
||
photo_urls: list[str] = []
|
||
for img in entity.get("mainImages") or []:
|
||
if img:
|
||
photo_urls.append(f"https:{img}" if img.startswith("//") else img)
|
||
if len(photo_urls) >= 5:
|
||
break
|
||
|
||
return ScrapedLot(
|
||
source="yandex",
|
||
source_url=source_url,
|
||
source_id=offer_id,
|
||
address=address,
|
||
lat=lat,
|
||
lon=lon,
|
||
rooms=rooms,
|
||
area_m2=area_m2,
|
||
living_area_m2=living_area_m2,
|
||
floor=floor,
|
||
total_floors=total_floors,
|
||
year_built=year_built,
|
||
house_type=house_type,
|
||
kitchen_area_m2=kitchen_area_m2,
|
||
ceiling_height_m=ceiling_height_m,
|
||
price_rub=price_rub,
|
||
price_per_m2=price_per_m2,
|
||
photo_urls=photo_urls,
|
||
house_source=house_source,
|
||
house_ext_id=house_ext_id,
|
||
house_url=None,
|
||
listing_segment=_segment_for(new_flat),
|
||
listing_date=listing_date,
|
||
publish_date=listing_date,
|
||
days_on_market=days_on_market,
|
||
description=description,
|
||
is_homeowner=is_homeowner,
|
||
is_pro_seller=is_pro_seller,
|
||
agency_name=agency_name,
|
||
metro_stations=metro_stations,
|
||
yandex_offer_id=offer_id,
|
||
predicted_price_rub=predicted_value,
|
||
predicted_price_min=predicted_min,
|
||
predicted_price_max=predicted_max,
|
||
price_trend=price_trend,
|
||
price_previous_rub=price_previous_rub,
|
||
raw_payload={
|
||
"page_param": page_param,
|
||
"ceiling_height": ceiling_height,
|
||
"kitchen_area_m2": kitchen_area_raw,
|
||
"raw_building_type": raw_building_type, # сырой SCREAMING — для отладки
|
||
"site_name": site_name,
|
||
"offer_id": offer_id,
|
||
},
|
||
)
|
||
except Exception:
|
||
logger.exception("yandex gate: entity parse failed offer_id=%s", entity.get("offerId"))
|
||
return None
|
||
|
||
|
||
class YandexRealtyScraper(BaseScraper):
|
||
name = "yandex"
|
||
base_url = "https://realty.yandex.ru"
|
||
request_delay_sec = 5.0
|
||
|
||
def __init__(self, city: str = DEFAULT_CITY) -> None:
|
||
super().__init__()
|
||
self.city = city
|
||
self.request_delay_sec = get_scraper_delay(self.name)
|
||
self._browser: BrowserFetcher | None = None
|
||
# _cffi_session retained only for _rotate_ip (changeip call).
|
||
self._cffi_session: _CurlCffiSession | None = None
|
||
self._cookies: dict[str, str] = {}
|
||
|
||
async def __aenter__(self) -> YandexRealtyScraper: # type: ignore[override]
|
||
"""Open ONE BrowserFetcher (camoufox) session, reused across all page fetches.
|
||
|
||
camoufox's real-browser fingerprint avoids the mobile-proxy IP tarpit
|
||
escalation that raw system-curl over SOCKS5 triggers. Opening per-page is
|
||
expensive and unnecessary.
|
||
|
||
fetch_timeout_s=_YANDEX_BROWSER_FETCH_TIMEOUT_S (30s) ограничивает потери
|
||
на один тайм-аутовый запрос: worst-case 30s×(1+retries)=90s на combo вместо 360s.
|
||
"""
|
||
self._browser = BrowserFetcher(
|
||
source="yandex", fetch_timeout_s=_YANDEX_BROWSER_FETCH_TIMEOUT_S
|
||
)
|
||
await self._browser.__aenter__()
|
||
return self
|
||
|
||
async def __aexit__(self, *args: object) -> None: # type: ignore[override]
|
||
if self._browser is not None:
|
||
await self._browser.__aexit__(*args)
|
||
self._browser = None
|
||
if self._cffi_session is not None:
|
||
await self._cffi_session.close()
|
||
self._cffi_session = None
|
||
|
||
async def _http_get(self, url: str, **kwargs: object) -> _CurlResponse: # type: ignore[override]
|
||
"""Fetch gate-API URL via the shared BrowserFetcher (camoufox).
|
||
|
||
camoufox returns HTML-wrapped JSON; the JSON is extracted and returned as
|
||
the body of a _CurlResponse so existing callers (json.loads(resp.text),
|
||
resp.status_code checks) keep working unchanged.
|
||
|
||
Maps outcomes onto the curl-style status contract:
|
||
- JSON extracted -> status_code=200, text=<json>
|
||
- fetch raised / no JSON found -> status_code=0 (treated as transient
|
||
failure by the caller retry logic)
|
||
"""
|
||
if self._browser is None:
|
||
raise RuntimeError("YandexRealtyScraper must be used as async context manager")
|
||
|
||
try:
|
||
content = await self._browser.fetch(url)
|
||
except Exception:
|
||
logger.warning("yandex gate: BrowserFetcher.fetch failed url=%s", url, exc_info=True)
|
||
return _CurlResponse(status_code=0, text="")
|
||
|
||
json_text = _extract_json_from_content(content)
|
||
if not json_text:
|
||
logger.warning(
|
||
"yandex gate: could not extract JSON from content url=%s first200=%r",
|
||
url,
|
||
content[:200],
|
||
)
|
||
return _CurlResponse(status_code=0, text="")
|
||
|
||
return _CurlResponse(status_code=200, text=json_text)
|
||
|
||
async def _rotate_ip(self) -> bool:
|
||
"""Rotate mobile proxy IP via changeip URL. Returns True on success."""
|
||
from app.core.config import settings as _settings
|
||
|
||
rotate_url = _settings.yandex_proxy_rotate_url or _settings.avito_proxy_rotate_url
|
||
if not rotate_url:
|
||
return False
|
||
sep = "&" if "?" in rotate_url else "?"
|
||
try:
|
||
async with _CurlCffiSession(timeout=30) as rot:
|
||
await rot.get(f"{rotate_url}{sep}format=json")
|
||
await asyncio.sleep(9)
|
||
logger.info("yandex proxy: IP rotated via changeip")
|
||
return True
|
||
except Exception:
|
||
logger.warning("yandex proxy: IP rotation failed", exc_info=True)
|
||
return False
|
||
|
||
def _build_url(
|
||
self,
|
||
page: int = 1,
|
||
rooms: str | None = None,
|
||
price_min: int | None = None,
|
||
price_max: int | None = None,
|
||
new_flat: str = "NO",
|
||
) -> str:
|
||
"""Build gate-API URL. page: 1-based (page=0 -> API error).
|
||
|
||
new_flat: "NO" -> vtorichka (default, preserves legacy behavior),
|
||
"YES" -> novostroyki (primary-sale / reassignment segment).
|
||
"""
|
||
params: dict[str, str | int] = {
|
||
"rgid": _EKB_RGID,
|
||
"type": "SELL",
|
||
"category": "APARTMENT",
|
||
"newFlat": new_flat,
|
||
"_pageType": "search",
|
||
"_providers": "react-search-results-data",
|
||
"page": page,
|
||
}
|
||
if rooms and rooms in ROOM_GATE_PARAM:
|
||
params["roomsTotal"] = ROOM_GATE_PARAM[rooms]
|
||
if price_min is not None:
|
||
params["priceMin"] = price_min
|
||
if price_max is not None:
|
||
params["priceMax"] = price_max
|
||
return f"{_GATE_URL}?{urlencode(params)}"
|
||
|
||
async def _fetch_page_json(
|
||
self,
|
||
rooms: str | None,
|
||
page: int,
|
||
price_min: int | None,
|
||
price_max: int | None,
|
||
new_flat: str = "NO",
|
||
) -> dict[str, Any] | None:
|
||
"""Fetch one gate-API page. Returns parsed JSON dict or None on error/invalid."""
|
||
url = self._build_url(
|
||
page=page, rooms=rooms, price_min=price_min, price_max=price_max, new_flat=new_flat
|
||
)
|
||
try:
|
||
resp = await self._http_get(url, timeout=60)
|
||
except Exception:
|
||
logger.exception("yandex gate: GET failed url=%s", url)
|
||
return None
|
||
if resp.status_code != 200: # type: ignore[union-attr]
|
||
logger.warning("yandex gate: HTTP %d url=%s", resp.status_code, url) # type: ignore[union-attr]
|
||
return None
|
||
try:
|
||
payload: dict[str, Any] = json.loads(resp.text) # type: ignore[union-attr]
|
||
except (json.JSONDecodeError, ValueError):
|
||
logger.warning("yandex gate: JSON parse failed url=%s", url)
|
||
return None
|
||
if _is_gate_error(payload):
|
||
logger.warning("yandex gate: error response url=%s keys=%s", url, list(payload.keys()))
|
||
return None
|
||
return payload
|
||
|
||
async def fetch_around(
|
||
self,
|
||
lat: float,
|
||
lon: float,
|
||
radius_m: int = 1000,
|
||
page: int = 1,
|
||
rooms: str | None = None,
|
||
price_min: int | None = None,
|
||
price_max: int | None = None,
|
||
new_flat: str = "NO",
|
||
) -> list[ScrapedLot]:
|
||
"""Fetch ONE page of gate-API results. lat/lon/radius_m ignored (uses rgid).
|
||
|
||
page: 1-based. Tarpit resilience: status_code==0 or JSON error
|
||
-> rotate IP + retry up to _YANDEX_TARPIT_MAX_RETRIES times.
|
||
Non-tarpit failures (4xx/5xx) are not retried.
|
||
new_flat: "NO" -> vtorichka, "YES" -> novostroyki.
|
||
"""
|
||
url = self._build_url(
|
||
page=page, rooms=rooms, price_min=price_min, price_max=price_max, new_flat=new_flat
|
||
)
|
||
|
||
payload: dict[str, Any] | None = None
|
||
for attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES):
|
||
try:
|
||
response = await self._http_get(url, timeout=60)
|
||
except Exception:
|
||
logger.exception("yandex gate fetch failed: %s", url)
|
||
return []
|
||
|
||
status = response.status_code # type: ignore[union-attr]
|
||
|
||
if status == 0:
|
||
logger.warning(
|
||
"yandex gate: status=0 (tarpit?) rooms=%s page=%d"
|
||
" -- rotating IP + retry attempt %d",
|
||
rooms,
|
||
page,
|
||
attempt + 1,
|
||
)
|
||
await self._rotate_ip()
|
||
await asyncio.sleep(2)
|
||
continue
|
||
|
||
if status != 200:
|
||
logger.warning(
|
||
"yandex gate: HTTP %d rooms=%s page=%d url=%s", status, rooms, page, url
|
||
)
|
||
return []
|
||
|
||
try:
|
||
payload = json.loads(response.text) # type: ignore[union-attr]
|
||
except (json.JSONDecodeError, ValueError):
|
||
logger.warning(
|
||
"yandex gate: JSON parse failed rooms=%s page=%d -- rotating + retry %d",
|
||
rooms,
|
||
page,
|
||
attempt + 1,
|
||
)
|
||
await self._rotate_ip()
|
||
await asyncio.sleep(2)
|
||
continue
|
||
|
||
if _is_gate_error(payload):
|
||
logger.warning(
|
||
"yandex gate: error payload rooms=%s page=%d keys=%s -- rotating + retry %d",
|
||
rooms,
|
||
page,
|
||
list(payload.keys()),
|
||
attempt + 1,
|
||
)
|
||
await self._rotate_ip()
|
||
await asyncio.sleep(2)
|
||
payload = None
|
||
continue
|
||
|
||
break
|
||
|
||
if payload is None:
|
||
logger.warning(
|
||
"yandex gate: all retries exhausted rooms=%s page=%d -- returning empty",
|
||
rooms,
|
||
page,
|
||
)
|
||
return []
|
||
|
||
lots = _parse_gate_json(payload, page_param=page, new_flat=new_flat)
|
||
logger.info(
|
||
"yandex gate page=%d rooms=%s price=[%s-%s] newFlat=%s: %d cards",
|
||
page,
|
||
rooms or "all",
|
||
price_min,
|
||
price_max,
|
||
new_flat,
|
||
len(lots),
|
||
)
|
||
await self.sleep_between_requests()
|
||
return lots
|
||
|
||
async def fetch_around_multi_room(
|
||
self,
|
||
lat: float,
|
||
lon: float,
|
||
radius_m: int = 1000,
|
||
max_pages: int = _GATE_MAX_PAGES_CAP,
|
||
rooms_list: list[str] | None = None,
|
||
price_ranges: list[tuple[int | None, int | None]] | None = None,
|
||
segments: list[str] | None = None,
|
||
on_combo: Any = None,
|
||
**_legacy_kwargs: Any,
|
||
) -> list[ScrapedLot]:
|
||
"""Fetch via segment x rooms x price-range combos; paginate each combo to totalPages.
|
||
|
||
Pagination driven by pager.totalPages from first page response.
|
||
Each combo iterates page=1 .. min(totalPages, max_pages, _GATE_MAX_PAGES_CAP).
|
||
Deduplicates by source_id/source_url across ALL combos AND segments (one `seen`).
|
||
Legacy mode (rooms_list=None, price_ranges=None): single citywide sweep.
|
||
segments: list of newFlat values, default ["NO"] (vtorichka only). Pass
|
||
["NO", "YES"] to sweep both vtorichka and novostroyki in one call.
|
||
|
||
on_combo: опциональный callback(combo_label: str, new_lots: list[ScrapedLot]) -> None.
|
||
Вызывается после каждого combo с новыми (не повторными) лотами из него.
|
||
Позволяет инкрементальный save: вызывающий код сохраняет new_lots сразу,
|
||
не дожидаясь конца sweep'а. Дубли между combo НЕ передаются повторно.
|
||
"""
|
||
seen: dict[str, ScrapedLot] = {}
|
||
_segments = segments or ["NO"]
|
||
combos_skipped = 0
|
||
|
||
if rooms_list is None and price_ranges is None:
|
||
combos: list[tuple[str | None, int | None, int | None]] = [(None, None, None)]
|
||
else:
|
||
r_list = rooms_list or list(ROOM_PATH.keys())
|
||
p_ranges = price_ranges or DEFAULT_PRICE_RANGES
|
||
combos = [(r, lo, hi) for r in r_list for lo, hi in p_ranges]
|
||
|
||
for new_flat in _segments:
|
||
_seg = _segment_for(new_flat)
|
||
for rooms, price_min, price_max in combos:
|
||
combo_label = f"{_seg}/{_combo_label(rooms, price_min, price_max)}"
|
||
total_pages: int | None = None
|
||
combo_new_lots: list[ScrapedLot] = []
|
||
combo_skipped = False
|
||
|
||
for page in range(1, max_pages + 1):
|
||
if page == 1:
|
||
payload_p1: dict[str, Any] | None = None
|
||
p1_url = self._build_url(
|
||
page=1,
|
||
rooms=rooms,
|
||
price_min=price_min,
|
||
price_max=price_max,
|
||
new_flat=new_flat,
|
||
)
|
||
for _attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES):
|
||
try:
|
||
resp = await self._http_get(p1_url, timeout=60)
|
||
except Exception:
|
||
logger.exception(
|
||
"yandex gate combo fetch failed combo=%s", combo_label
|
||
)
|
||
break
|
||
if resp.status_code == 0: # type: ignore[union-attr]
|
||
logger.warning(
|
||
"yandex gate: tarpit combo=%s page=1 attempt=%d -- rotating",
|
||
combo_label,
|
||
_attempt + 1,
|
||
)
|
||
await self._rotate_ip()
|
||
await asyncio.sleep(2)
|
||
continue
|
||
if resp.status_code != 200: # type: ignore[union-attr]
|
||
logger.warning(
|
||
"yandex gate: HTTP %d combo=%s page=1",
|
||
resp.status_code,
|
||
combo_label, # type: ignore[union-attr]
|
||
)
|
||
break
|
||
try:
|
||
payload_p1 = json.loads(resp.text) # type: ignore[union-attr]
|
||
except (json.JSONDecodeError, ValueError):
|
||
logger.warning(
|
||
"yandex gate: JSON parse error combo=%s page=1", combo_label
|
||
)
|
||
payload_p1 = None
|
||
await self._rotate_ip()
|
||
await asyncio.sleep(2)
|
||
continue
|
||
if _is_gate_error(payload_p1):
|
||
logger.warning(
|
||
"yandex gate: error payload combo=%s page=1", combo_label
|
||
)
|
||
payload_p1 = None
|
||
break
|
||
break
|
||
|
||
if payload_p1 is None:
|
||
logger.warning(
|
||
"yandex gate combo [%s] page=1: all retries exhausted -- skipping",
|
||
combo_label,
|
||
)
|
||
combo_skipped = True
|
||
combos_skipped += 1
|
||
break
|
||
|
||
result = _extract_gate_data(payload_p1)
|
||
if result is None:
|
||
combo_skipped = True
|
||
combos_skipped += 1
|
||
break
|
||
_entities_p1, pager_p1 = result
|
||
total_pages = min(
|
||
pager_p1.get("totalPages", 1),
|
||
max_pages,
|
||
_GATE_MAX_PAGES_CAP,
|
||
)
|
||
lots_p1 = _parse_gate_json(payload_p1, page_param=1, new_flat=new_flat)
|
||
if not lots_p1:
|
||
logger.debug(
|
||
"yandex gate combo [%s] page=1: empty -- stopping", combo_label
|
||
)
|
||
break
|
||
for lot in lots_p1:
|
||
key = lot.source_id or lot.source_url
|
||
if key and key not in seen:
|
||
seen[key] = lot
|
||
combo_new_lots.append(lot)
|
||
await self.sleep_between_requests()
|
||
if total_pages <= 1:
|
||
break
|
||
continue
|
||
|
||
if total_pages is not None and page > total_pages:
|
||
break
|
||
|
||
lots = await self.fetch_around(
|
||
lat,
|
||
lon,
|
||
radius_m,
|
||
page=page,
|
||
rooms=rooms,
|
||
price_min=price_min,
|
||
price_max=price_max,
|
||
new_flat=new_flat,
|
||
)
|
||
if not lots:
|
||
logger.debug(
|
||
"yandex gate combo [%s] page=%d: empty -- stopping",
|
||
combo_label,
|
||
page,
|
||
)
|
||
break
|
||
for lot in lots:
|
||
key = lot.source_id or lot.source_url
|
||
if key and key not in seen:
|
||
seen[key] = lot
|
||
combo_new_lots.append(lot)
|
||
|
||
# Инкрементальный save: вызываем on_combo если есть новые лоты.
|
||
# Скипнутые combo (combo_skipped=True) не вызывают on_combo.
|
||
if on_combo is not None and combo_new_lots and not combo_skipped:
|
||
try:
|
||
on_combo(combo_label, combo_new_lots)
|
||
except Exception:
|
||
logger.exception(
|
||
"yandex gate: on_combo callback failed combo=%s", combo_label
|
||
)
|
||
|
||
logger.info(
|
||
"yandex gate aggregate: %d unique lots (%d combos x %d segments=%s, skipped=%d)",
|
||
len(seen),
|
||
len(combos),
|
||
len(_segments),
|
||
_segments,
|
||
combos_skipped,
|
||
)
|
||
return list(seen.values())
|
||
|
||
async def fetch_all_secondary(
|
||
self,
|
||
*,
|
||
rooms_buckets: list[str] | None = None,
|
||
price_cap_per_bucket: int = 500,
|
||
max_pages_per_bucket: int = _GATE_MAX_PAGES_CAP,
|
||
concurrency: int = 4,
|
||
on_bucket: Any = None,
|
||
on_progress: Any = None,
|
||
skip_buckets: set[str] | None = None,
|
||
) -> list[ScrapedLot]:
|
||
"""Exhaustive EKB vtorichka load via KOMNATNOST x TSENA partitioning.
|
||
|
||
Uses gate-API JSON (totalItems from pager) instead of DOM state extraction.
|
||
Стартует с общей data-driven seed-сетки ценовых брекетов
|
||
(get_price_seed_brackets() — shared EKB grid, общая с avito/cian) вместо
|
||
одной корень-бисекции [0, _YANDEX_MAX_PRICE]. Закрытые seed-брекеты передают
|
||
hi = br_hi - 1 (priceMax эксклюзивный сверху → нет пересечений), открытый
|
||
(br_hi is None) — hi=None (без priceMax, весь хвост люкса). Дедуп по
|
||
source_id в общем seen страхует на стыках.
|
||
"""
|
||
_buckets = rooms_buckets if rooms_buckets is not None else list(ROOM_PATH.keys())
|
||
seen: dict[str, ScrapedLot] = {}
|
||
|
||
for rooms_key in _buckets:
|
||
logger.info(
|
||
"yandex gate exhaustive: starting rooms=%s (price_cap=%d concurrency=%d)",
|
||
rooms_key,
|
||
price_cap_per_bucket,
|
||
concurrency,
|
||
)
|
||
before = len(seen)
|
||
for br_lo, br_hi in get_price_seed_brackets():
|
||
walk_hi = br_hi - 1 if br_hi is not None else None
|
||
await self._walk_price_range(
|
||
rooms=rooms_key,
|
||
lo=br_lo,
|
||
hi=walk_hi,
|
||
seen=seen,
|
||
price_cap_per_bucket=price_cap_per_bucket,
|
||
max_pages_per_bucket=max_pages_per_bucket,
|
||
concurrency=concurrency,
|
||
on_bucket=on_bucket,
|
||
skip_buckets=skip_buckets,
|
||
)
|
||
room_collected = len(seen) - before
|
||
logger.info(
|
||
"yandex gate exhaustive: rooms=%s done -- collected %d (total unique=%d)",
|
||
rooms_key,
|
||
room_collected,
|
||
len(seen),
|
||
)
|
||
if on_progress is not None:
|
||
on_progress(len(seen))
|
||
|
||
logger.info("yandex gate exhaustive: DONE -- total unique=%d lots", len(seen))
|
||
return list(seen.values())
|
||
|
||
async def _walk_price_range(
|
||
self,
|
||
*,
|
||
rooms: str | None,
|
||
lo: int,
|
||
hi: int | None,
|
||
seen: dict[str, ScrapedLot],
|
||
price_cap_per_bucket: int,
|
||
max_pages_per_bucket: int,
|
||
concurrency: int = 4,
|
||
on_bucket: Any = None,
|
||
skip_buckets: set[str] | None = None,
|
||
_depth: int = 0,
|
||
) -> None:
|
||
"""Recursive binary price-range partitioning.
|
||
|
||
Probe page=1 -> read totalItems from pager.
|
||
Closed bracket (hi is not None): totalItems > cap and bracket >= MIN_BRACKET
|
||
-> split; else paginate leaf.
|
||
Open bracket (hi is None): top seed-bracket without priceMax ceiling. Cannot
|
||
split (no mid) -> paginate leaf directly (lux tail tiny, tail-loss accepted).
|
||
Fallback: totalItems=None -> paginate-until-empty (safe degradation).
|
||
skip_buckets: resume checkpoint -- skip leaf buckets already done.
|
||
"""
|
||
_lo_param = lo if lo > 0 else None
|
||
_hi_repr = "open" if hi is None else str(hi)
|
||
|
||
probe_payload = await self._fetch_page_json(rooms, 1, _lo_param, hi)
|
||
await asyncio.sleep(self.request_delay_sec)
|
||
|
||
total: int | None = None
|
||
probe_lots: list[ScrapedLot] = []
|
||
|
||
if probe_payload is None:
|
||
logger.warning(
|
||
"yandex gate: probe failed rooms=%s [%d, %s] depth=%d -- rotating + retry",
|
||
rooms,
|
||
lo,
|
||
_hi_repr,
|
||
_depth,
|
||
)
|
||
rotated = await self._rotate_ip()
|
||
if rotated:
|
||
probe_payload = await self._fetch_page_json(rooms, 1, _lo_param, hi)
|
||
await asyncio.sleep(self.request_delay_sec)
|
||
|
||
if probe_payload is not None:
|
||
result = _extract_gate_data(probe_payload)
|
||
if result is not None:
|
||
_, pager = result
|
||
total = pager.get("totalItems")
|
||
probe_lots = _parse_gate_json(probe_payload, page_param=1)
|
||
else:
|
||
logger.warning(
|
||
"yandex gate: probe data extract failed rooms=%s [%d, %s]", rooms, lo, _hi_repr
|
||
)
|
||
|
||
if total is None:
|
||
# Degraded path: paginate-until-empty without knowing total
|
||
logger.warning(
|
||
"yandex gate: totalItems unknown rooms=%s [%d, %s] -- paginate until empty",
|
||
rooms,
|
||
lo,
|
||
_hi_repr,
|
||
)
|
||
page = 1
|
||
bucket_key = _combo_label(rooms, lo, hi)
|
||
if skip_buckets and bucket_key in skip_buckets:
|
||
return
|
||
pages_fetched = 0
|
||
while pages_fetched < max_pages_per_bucket:
|
||
payload = await self._fetch_page_json(rooms, page, _lo_param, hi)
|
||
await asyncio.sleep(self.request_delay_sec)
|
||
if payload is None:
|
||
break
|
||
lots = _parse_gate_json(payload, page_param=page)
|
||
if not lots:
|
||
break
|
||
for lot in lots:
|
||
if lot.source_id and lot.source_id not in seen:
|
||
seen[lot.source_id] = lot
|
||
page += 1
|
||
pages_fetched += 1
|
||
if on_bucket is not None:
|
||
on_bucket(bucket_key, len(seen))
|
||
return
|
||
|
||
# Split or paginate. ОТКРЫТЫЙ брекет (hi is None) делить нельзя (нет mid) —
|
||
# сразу пагинируем leaf напрямую (хвост люкса крошечный, tail-loss accepted).
|
||
cap = price_cap_per_bucket
|
||
_min_bracket = 500_000
|
||
|
||
if hi is not None and total > cap and (hi - lo) >= _min_bracket and _depth < 8:
|
||
mid = (lo + hi) // 2
|
||
await self._walk_price_range(
|
||
rooms=rooms,
|
||
lo=lo,
|
||
hi=mid,
|
||
seen=seen,
|
||
price_cap_per_bucket=cap,
|
||
max_pages_per_bucket=max_pages_per_bucket,
|
||
concurrency=concurrency,
|
||
on_bucket=on_bucket,
|
||
skip_buckets=skip_buckets,
|
||
_depth=_depth + 1,
|
||
)
|
||
await self._walk_price_range(
|
||
rooms=rooms,
|
||
lo=mid + 1,
|
||
hi=hi,
|
||
seen=seen,
|
||
price_cap_per_bucket=cap,
|
||
max_pages_per_bucket=max_pages_per_bucket,
|
||
concurrency=concurrency,
|
||
on_bucket=on_bucket,
|
||
skip_buckets=skip_buckets,
|
||
_depth=_depth + 1,
|
||
)
|
||
return
|
||
|
||
# Leaf bucket: paginate
|
||
bucket_key = _combo_label(rooms, lo, hi)
|
||
if skip_buckets and bucket_key in skip_buckets:
|
||
logger.debug("yandex gate: skip bucket %s (checkpoint)", bucket_key)
|
||
return
|
||
|
||
total_pages = min(
|
||
math.ceil(total / 20),
|
||
_GATE_MAX_PAGES_CAP,
|
||
max_pages_per_bucket,
|
||
)
|
||
logger.info("yandex gate: leaf bucket %s total=%d pages=%d", bucket_key, total, total_pages)
|
||
|
||
# Add probe lots (page 1 already fetched)
|
||
for lot in probe_lots:
|
||
if lot.source_id and lot.source_id not in seen:
|
||
seen[lot.source_id] = lot
|
||
|
||
if total_pages <= 1:
|
||
if on_bucket is not None:
|
||
on_bucket(bucket_key, len(seen))
|
||
return
|
||
|
||
# Paginate pages 2..total_pages with concurrency
|
||
sem = asyncio.Semaphore(concurrency)
|
||
|
||
async def _fetch_leaf_page(pg: int) -> list[ScrapedLot]:
|
||
async with sem:
|
||
payload = await self._fetch_page_json(rooms, pg, _lo_param, hi)
|
||
await asyncio.sleep(self.request_delay_sec)
|
||
if payload is None:
|
||
return []
|
||
return _parse_gate_json(payload, page_param=pg)
|
||
|
||
tasks = [_fetch_leaf_page(pg) for pg in range(2, total_pages + 1)]
|
||
results = await asyncio.gather(*tasks)
|
||
for page_lots in results:
|
||
for lot in page_lots:
|
||
if lot.source_id and lot.source_id not in seen:
|
||
seen[lot.source_id] = lot
|
||
|
||
if on_bucket is not None:
|
||
on_bucket(bucket_key, len(seen))
|
||
|
||
|
||
def _combo_label(rooms: str | None, lo: int | None, hi: int | None) -> str:
|
||
"""Уникальная метка бакета для checkpoint-логики."""
|
||
return f"{rooms or 'any'}:{lo}-{hi}"
|