1015 lines
38 KiB
Python
1015 lines
38 KiB
Python
"""Yandex.Nedvizhimost scraper (realty.yandex.ru) -- gate-API JSON parser.
|
|
|
|
Transport: BrowserFetcher (camoufox real-browser fingerprint via tradein-browser service).
|
|
camoufox's fingerprint is not tarpitted by yandex and does not burn the mobile-proxy IP;
|
|
raw system-curl over SOCKS5 gets the proxy IP flagged. One BrowserFetcher context is opened
|
|
per scraper session (__aenter__) and reused across all page fetches.
|
|
|
|
camoufox returns the gate-API JSON wrapped in HTML (verified prod 2026-06-17):
|
|
<html><head>...</head><body><pre>{"response":...}</pre></body></html>
|
|
JSON is extracted via _extract_json_from_content() (handles the <pre> wrapper).
|
|
|
|
API path: GET https://realty.yandex.ru/gate/react-page/get/
|
|
_pageType=search&_providers=react-search-results-data
|
|
type=SELL&category=APARTMENT&newFlat=NO&rgid=559132 (EKB)
|
|
roomsTotal=<int|STUDIO>&priceMin=...&priceMax=...&page=<1-based>
|
|
|
|
Response shape: response.search.offers.{entities[], pager{page,pageSize,totalItems,totalPages}}
|
|
|
|
Page param is 1-based: page=1 -> first page (pager.page=0 in response).
|
|
page=0 -> API returns error. Paginate from page=1 to totalPages (inclusive).
|
|
|
|
Studio roomsTotal:
|
|
Request: roomsTotal=STUDIO
|
|
Response entities: roomsTotal=None -> mapped to rooms=0 (existing convention).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import math
|
|
import re
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, date, datetime
|
|
from typing import Any
|
|
from urllib.parse import urlencode
|
|
|
|
from curl_cffi.requests import AsyncSession as _CurlCffiSession
|
|
|
|
from app.services.scraper_settings import get_scraper_delay
|
|
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
|
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# camoufox renders JSON endpoints as <pre>{"response":...}</pre> inside an HTML wrapper.
|
|
_RE_PRE_JSON = re.compile("<pre[^>]*>(\\{.*?)</pre>", re.DOTALL | re.IGNORECASE)
|
|
|
|
DEFAULT_CITY = "ekaterinburg"
|
|
_GATE_MAX_PAGES_CAP = 50
|
|
_EKB_RGID = 559132
|
|
_GATE_URL = "https://realty.yandex.ru/gate/react-page/get/"
|
|
|
|
ROOM_GATE_PARAM: dict[str, str | int] = {
|
|
"studio": "STUDIO",
|
|
"1": 1,
|
|
"2": 2,
|
|
"3": 3,
|
|
"4+": 4,
|
|
}
|
|
|
|
ROOM_PATH: dict[str, str] = {
|
|
"studio": "studiya",
|
|
"1": "odnokomnatnaya",
|
|
"2": "dvuhkomnatnaya",
|
|
"3": "trehkomnatnaya",
|
|
"4+": "4-i-bolee",
|
|
}
|
|
|
|
DEFAULT_PRICE_RANGES: list[tuple[int | None, int | None]] = [
|
|
(None, 5_000_000),
|
|
(5_000_000, 7_000_000),
|
|
(7_000_000, 10_000_000),
|
|
(10_000_000, 15_000_000),
|
|
(15_000_000, 25_000_000),
|
|
(25_000_000, None),
|
|
]
|
|
|
|
_YANDEX_MAX_PRICE = 200_000_000
|
|
_YANDEX_MIN_BRACKET = 50_000
|
|
_GATE_PAGE_SIZE = 20
|
|
_YANDEX_TARPIT_MAX_RETRIES: int = 2
|
|
|
|
|
|
@dataclass
|
|
class _CurlResponse:
|
|
status_code: int
|
|
text: str
|
|
|
|
|
|
def _is_gate_error(payload: dict[str, Any]) -> bool:
|
|
"""Return True when gate-API returned an error object instead of search results."""
|
|
return "error" in payload or "response" not in payload
|
|
|
|
|
|
def _extract_gate_data(
|
|
payload: dict[str, Any],
|
|
) -> tuple[list[dict[str, Any]], dict[str, Any]] | None:
|
|
"""Extract (entities, pager) from valid gate-API response. None on missing path."""
|
|
try:
|
|
offers = payload["response"]["search"]["offers"]
|
|
entities: list[dict[str, Any]] = offers["entities"]
|
|
pager: dict[str, Any] = offers["pager"]
|
|
return entities, pager
|
|
except (KeyError, TypeError):
|
|
return None
|
|
|
|
|
|
def _extract_json_from_content(content: str) -> str | None:
|
|
"""Extract raw JSON text from camoufox HTML-wrapped content.
|
|
|
|
camoufox renders a JSON endpoint as:
|
|
<html><head>...</head><body><pre>{"response":...}</pre></body></html>
|
|
|
|
Strategy:
|
|
1. Try <pre>...</pre> regex (primary -- matches prod output).
|
|
2. Fallback: slice from the first '{' (handles raw-JSON response, no wrapper).
|
|
|
|
Returns the JSON string, or None if no '{' is found.
|
|
"""
|
|
m = _RE_PRE_JSON.search(content)
|
|
if m:
|
|
return m.group(1)
|
|
|
|
idx = content.find("{")
|
|
if idx != -1:
|
|
return content[idx:]
|
|
|
|
return None
|
|
|
|
|
|
def _segment_for(new_flat: str) -> str:
|
|
"""Map a newFlat gate-API value to the listings.listing_segment value."""
|
|
return "novostroyki" if new_flat == "YES" else "vtorichka"
|
|
|
|
|
|
def _parse_creation_date(raw: Any) -> tuple[date | None, int | None]:
|
|
"""Parse gate-API creationDate ('2026-05-24T15:10:27Z') → (date, days_on_market).
|
|
|
|
days_on_market = floor((now - creationDate).days), clamped to >= 0.
|
|
Returns (None, None) when raw is absent or unparseable (never raises).
|
|
"""
|
|
if not raw or not isinstance(raw, str):
|
|
return None, None
|
|
txt = raw.strip()
|
|
# datetime.fromisoformat handles 'Z' only from py3.11+, but normalise defensively.
|
|
if txt.endswith("Z"):
|
|
txt = txt[:-1] + "+00:00"
|
|
try:
|
|
dt = datetime.fromisoformat(txt)
|
|
except ValueError:
|
|
return None, None
|
|
if dt.tzinfo is None:
|
|
dt = dt.replace(tzinfo=UTC)
|
|
listing_date = dt.date()
|
|
delta_days = (datetime.now(UTC) - dt).days
|
|
days_on_market = max(delta_days, 0)
|
|
return listing_date, days_on_market
|
|
|
|
|
|
def _author_fields(author: dict[str, Any]) -> tuple[bool | None, bool | None, str | None]:
|
|
"""Map entity.author → (is_homeowner, is_pro_seller, agency_name).
|
|
|
|
author.category ∈ {OWNER, AGENCY, AGENT, ...}.
|
|
OWNER → is_homeowner=True
|
|
AGENCY / AGENT → is_pro_seller=True, agency_name = organization or agentName
|
|
Returns (None, None, None) when author/category absent.
|
|
"""
|
|
category = (author.get("category") or "").upper()
|
|
if not category:
|
|
return None, None, None
|
|
if category == "OWNER":
|
|
return True, None, None
|
|
if category in {"AGENCY", "AGENT"}:
|
|
agency_name = author.get("organization") or author.get("agentName") or None
|
|
return None, True, agency_name
|
|
return None, None, None
|
|
|
|
|
|
def _to_bigint(raw: Any) -> int | None:
|
|
"""Coerce a gate-API numeric value (str or number) to int. None when unparseable."""
|
|
if raw is None:
|
|
return None
|
|
try:
|
|
return int(float(raw))
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _predicted_prices(entity: dict[str, Any]) -> tuple[int | None, int | None, int | None]:
|
|
"""Extract (value, min, max) from entity.predictedPrice.predictedPrice.
|
|
|
|
Yandex's own per-offer valuation. Fields are strings (e.g. "7554000").
|
|
Returns (None, None, None) when the nested path is absent.
|
|
"""
|
|
outer = entity.get("predictedPrice") or {}
|
|
inner = outer.get("predictedPrice") or {}
|
|
if not isinstance(inner, dict):
|
|
return None, None, None
|
|
return (
|
|
_to_bigint(inner.get("value")),
|
|
_to_bigint(inner.get("min")),
|
|
_to_bigint(inner.get("max")),
|
|
)
|
|
|
|
|
|
def _metro_stations(location: dict[str, Any]) -> list[dict[str, Any]]:
|
|
"""Build metro_stations [{name, min}] from location.metroList[] (+ location.metro)."""
|
|
stations: list[dict[str, Any]] = []
|
|
raw_list = location.get("metroList")
|
|
if isinstance(raw_list, list):
|
|
for m in raw_list:
|
|
if isinstance(m, dict) and m.get("name"):
|
|
stations.append({"name": m.get("name"), "min": m.get("timeToMetro")})
|
|
if not stations:
|
|
single = location.get("metro")
|
|
if isinstance(single, dict) and single.get("name"):
|
|
stations.append({"name": single.get("name"), "min": single.get("timeToMetro")})
|
|
return stations
|
|
|
|
|
|
def _parse_gate_json(
|
|
payload: dict[str, Any], page_param: int = 1, new_flat: str = "NO"
|
|
) -> list[ScrapedLot]:
|
|
"""Parse gate-API JSON payload into a list of ScrapedLot.
|
|
|
|
Verified field mapping (from .issdump/yandex_gate_api_sample.json + live prod 2026-06-17):
|
|
offerId -> source_id
|
|
url (//realty.yandex.ru/offer/) -> source_url (prepend "https:")
|
|
price.value -> price_rub
|
|
price.valuePerPart -> price_per_m2
|
|
area.value -> area_m2
|
|
livingSpace.value -> living_area_m2
|
|
kitchenSpace.value -> raw_payload["kitchen_area_m2"]
|
|
roomsTotal (int | None) -> rooms (None -> 0 for studio)
|
|
floorsOffered[0] -> floor
|
|
floorsTotal -> total_floors
|
|
building.builtYear -> year_built
|
|
building.buildingType -> house_type (raw: "MONOLIT"/"BRICK"/etc.)
|
|
ceilingHeight -> raw_payload["ceiling_height"]
|
|
location.point.latitude/longitude -> lat / lon
|
|
location.geocoderAddress -> address (full, with house number)
|
|
mainImages[] -> photo_urls (prepend "https:", up to 5)
|
|
building.siteId -> house_ext_id (JK linkage)
|
|
listing_segment -> "novostroyki" (newFlat=YES) | "vtorichka" (newFlat=NO)
|
|
creationDate (ISO Z) -> listing_date / publish_date + days_on_market
|
|
description -> description
|
|
author.category/organization -> is_homeowner / is_pro_seller / agency_name
|
|
predictedPrice.predictedPrice.* -> predicted_price_rub / _min / _max
|
|
price.trend / price.previous -> price_trend / price_previous_rub
|
|
location.metroList[] -> metro_stations [{name, min}]
|
|
offerId -> yandex_offer_id
|
|
"""
|
|
result = _extract_gate_data(payload)
|
|
if result is None:
|
|
logger.warning("yandex gate: response missing response.search.offers path")
|
|
return []
|
|
|
|
entities, _pager = result
|
|
lots: list[ScrapedLot] = []
|
|
for entity in entities:
|
|
lot = _entity_to_lot(entity, page_param=page_param, new_flat=new_flat)
|
|
if lot is not None:
|
|
lots.append(lot)
|
|
return lots
|
|
|
|
|
|
def _entity_to_lot(
|
|
entity: dict[str, Any], page_param: int = 1, new_flat: str = "NO"
|
|
) -> ScrapedLot | None:
|
|
"""Convert one gate-API entity dict to ScrapedLot. None on missing required fields."""
|
|
try:
|
|
offer_id = str(entity.get("offerId") or "")
|
|
if not offer_id:
|
|
return None
|
|
|
|
url_raw = entity.get("url") or ""
|
|
source_url = f"https:{url_raw}" if url_raw.startswith("//") else url_raw
|
|
if not source_url:
|
|
source_url = f"https://realty.yandex.ru/offer/{offer_id}/"
|
|
|
|
price_dict = entity.get("price") or {}
|
|
price_val = price_dict.get("value")
|
|
if not price_val or price_val <= 0:
|
|
return None
|
|
price_rub = int(price_val)
|
|
|
|
price_per_m2_raw = price_dict.get("valuePerPart")
|
|
price_per_m2 = int(price_per_m2_raw) if price_per_m2_raw else None
|
|
|
|
area_dict = entity.get("area") or {}
|
|
area_m2_raw = area_dict.get("value")
|
|
area_m2 = float(area_m2_raw) if area_m2_raw is not None else None
|
|
|
|
living_dict = entity.get("livingSpace") or {}
|
|
living_area_raw = living_dict.get("value")
|
|
living_area_m2 = float(living_area_raw) if living_area_raw is not None else None
|
|
|
|
kitchen_dict = entity.get("kitchenSpace") or {}
|
|
kitchen_area_raw = kitchen_dict.get("value")
|
|
|
|
rooms_raw = entity.get("roomsTotal")
|
|
if rooms_raw is None:
|
|
rooms = 0 # studio
|
|
else:
|
|
try:
|
|
rooms = int(rooms_raw)
|
|
except (TypeError, ValueError):
|
|
rooms = None
|
|
|
|
floors_offered = entity.get("floorsOffered") or []
|
|
floor = int(floors_offered[0]) if floors_offered else None
|
|
floors_total_raw = entity.get("floorsTotal")
|
|
total_floors = int(floors_total_raw) if floors_total_raw is not None else None
|
|
|
|
ceiling_height = entity.get("ceilingHeight")
|
|
|
|
building = entity.get("building") or {}
|
|
year_built_raw = building.get("builtYear")
|
|
year_built = int(year_built_raw) if year_built_raw is not None else None
|
|
house_type = building.get("buildingType")
|
|
|
|
site_id = building.get("siteId")
|
|
site_name = building.get("siteName")
|
|
house_source: str | None = None
|
|
house_ext_id: str | None = None
|
|
if site_id:
|
|
house_source = "yandex_realty_nb"
|
|
house_ext_id = str(site_id)
|
|
|
|
location = entity.get("location") or {}
|
|
point = location.get("point") or {}
|
|
lat_raw = point.get("latitude")
|
|
lon_raw = point.get("longitude")
|
|
lat = float(lat_raw) if lat_raw is not None else None
|
|
lon = float(lon_raw) if lon_raw is not None else None
|
|
|
|
address = location.get("geocoderAddress") or location.get("streetAddress") or None
|
|
|
|
# ── Rich gate-API fields ───────────────────────────────────────────
|
|
listing_date, days_on_market = _parse_creation_date(entity.get("creationDate"))
|
|
description = entity.get("description") or None
|
|
|
|
author = entity.get("author") or {}
|
|
is_homeowner, is_pro_seller, agency_name = _author_fields(author)
|
|
|
|
predicted_value, predicted_min, predicted_max = _predicted_prices(entity)
|
|
|
|
price_trend = price_dict.get("trend") or None
|
|
price_previous_rub = _to_bigint(price_dict.get("previous"))
|
|
|
|
metro_stations = _metro_stations(location)
|
|
|
|
photo_urls: list[str] = []
|
|
for img in entity.get("mainImages") or []:
|
|
if img:
|
|
photo_urls.append(f"https:{img}" if img.startswith("//") else img)
|
|
if len(photo_urls) >= 5:
|
|
break
|
|
|
|
return ScrapedLot(
|
|
source="yandex",
|
|
source_url=source_url,
|
|
source_id=offer_id,
|
|
address=address,
|
|
lat=lat,
|
|
lon=lon,
|
|
rooms=rooms,
|
|
area_m2=area_m2,
|
|
living_area_m2=living_area_m2,
|
|
floor=floor,
|
|
total_floors=total_floors,
|
|
year_built=year_built,
|
|
house_type=house_type,
|
|
price_rub=price_rub,
|
|
price_per_m2=price_per_m2,
|
|
photo_urls=photo_urls,
|
|
house_source=house_source,
|
|
house_ext_id=house_ext_id,
|
|
house_url=None,
|
|
listing_segment=_segment_for(new_flat),
|
|
listing_date=listing_date,
|
|
publish_date=listing_date,
|
|
days_on_market=days_on_market,
|
|
description=description,
|
|
is_homeowner=is_homeowner,
|
|
is_pro_seller=is_pro_seller,
|
|
agency_name=agency_name,
|
|
metro_stations=metro_stations,
|
|
yandex_offer_id=offer_id,
|
|
predicted_price_rub=predicted_value,
|
|
predicted_price_min=predicted_min,
|
|
predicted_price_max=predicted_max,
|
|
price_trend=price_trend,
|
|
price_previous_rub=price_previous_rub,
|
|
raw_payload={
|
|
"page_param": page_param,
|
|
"ceiling_height": ceiling_height,
|
|
"kitchen_area_m2": kitchen_area_raw,
|
|
"site_name": site_name,
|
|
"offer_id": offer_id,
|
|
},
|
|
)
|
|
except Exception:
|
|
logger.exception("yandex gate: entity parse failed offer_id=%s", entity.get("offerId"))
|
|
return None
|
|
|
|
|
|
class YandexRealtyScraper(BaseScraper):
|
|
name = "yandex"
|
|
base_url = "https://realty.yandex.ru"
|
|
request_delay_sec = 5.0
|
|
|
|
def __init__(self, city: str = DEFAULT_CITY) -> None:
|
|
super().__init__()
|
|
self.city = city
|
|
self.request_delay_sec = get_scraper_delay(self.name)
|
|
self._browser: BrowserFetcher | None = None
|
|
# _cffi_session retained only for _rotate_ip (changeip call).
|
|
self._cffi_session: _CurlCffiSession | None = None
|
|
self._cookies: dict[str, str] = {}
|
|
|
|
async def __aenter__(self) -> YandexRealtyScraper: # type: ignore[override]
|
|
"""Open ONE BrowserFetcher (camoufox) session, reused across all page fetches.
|
|
|
|
camoufox's real-browser fingerprint avoids the mobile-proxy IP tarpit
|
|
escalation that raw system-curl over SOCKS5 triggers. Opening per-page is
|
|
expensive and unnecessary.
|
|
"""
|
|
self._browser = BrowserFetcher()
|
|
await self._browser.__aenter__()
|
|
return self
|
|
|
|
async def __aexit__(self, *args: object) -> None: # type: ignore[override]
|
|
if self._browser is not None:
|
|
await self._browser.__aexit__(*args)
|
|
self._browser = None
|
|
if self._cffi_session is not None:
|
|
await self._cffi_session.close()
|
|
self._cffi_session = None
|
|
|
|
async def _http_get(self, url: str, **kwargs: object) -> _CurlResponse: # type: ignore[override]
|
|
"""Fetch gate-API URL via the shared BrowserFetcher (camoufox).
|
|
|
|
camoufox returns HTML-wrapped JSON; the JSON is extracted and returned as
|
|
the body of a _CurlResponse so existing callers (json.loads(resp.text),
|
|
resp.status_code checks) keep working unchanged.
|
|
|
|
Maps outcomes onto the curl-style status contract:
|
|
- JSON extracted -> status_code=200, text=<json>
|
|
- fetch raised / no JSON found -> status_code=0 (treated as transient
|
|
failure by the caller retry logic)
|
|
"""
|
|
if self._browser is None:
|
|
raise RuntimeError("YandexRealtyScraper must be used as async context manager")
|
|
|
|
try:
|
|
content = await self._browser.fetch(url)
|
|
except Exception:
|
|
logger.warning("yandex gate: BrowserFetcher.fetch failed url=%s", url, exc_info=True)
|
|
return _CurlResponse(status_code=0, text="")
|
|
|
|
json_text = _extract_json_from_content(content)
|
|
if not json_text:
|
|
logger.warning(
|
|
"yandex gate: could not extract JSON from content url=%s first200=%r",
|
|
url,
|
|
content[:200],
|
|
)
|
|
return _CurlResponse(status_code=0, text="")
|
|
|
|
return _CurlResponse(status_code=200, text=json_text)
|
|
|
|
async def _rotate_ip(self) -> bool:
|
|
"""Rotate mobile proxy IP via changeip URL. Returns True on success."""
|
|
from app.core.config import settings as _settings
|
|
|
|
rotate_url = _settings.yandex_proxy_rotate_url or _settings.avito_proxy_rotate_url
|
|
if not rotate_url:
|
|
return False
|
|
sep = "&" if "?" in rotate_url else "?"
|
|
try:
|
|
async with _CurlCffiSession(timeout=30) as rot:
|
|
await rot.get(f"{rotate_url}{sep}format=json")
|
|
await asyncio.sleep(9)
|
|
logger.info("yandex proxy: IP rotated via changeip")
|
|
return True
|
|
except Exception:
|
|
logger.warning("yandex proxy: IP rotation failed", exc_info=True)
|
|
return False
|
|
|
|
def _build_url(
|
|
self,
|
|
page: int = 1,
|
|
rooms: str | None = None,
|
|
price_min: int | None = None,
|
|
price_max: int | None = None,
|
|
new_flat: str = "NO",
|
|
) -> str:
|
|
"""Build gate-API URL. page: 1-based (page=0 -> API error).
|
|
|
|
new_flat: "NO" -> vtorichka (default, preserves legacy behavior),
|
|
"YES" -> novostroyki (primary-sale / reassignment segment).
|
|
"""
|
|
params: dict[str, str | int] = {
|
|
"rgid": _EKB_RGID,
|
|
"type": "SELL",
|
|
"category": "APARTMENT",
|
|
"newFlat": new_flat,
|
|
"_pageType": "search",
|
|
"_providers": "react-search-results-data",
|
|
"page": page,
|
|
}
|
|
if rooms and rooms in ROOM_GATE_PARAM:
|
|
params["roomsTotal"] = ROOM_GATE_PARAM[rooms]
|
|
if price_min is not None:
|
|
params["priceMin"] = price_min
|
|
if price_max is not None:
|
|
params["priceMax"] = price_max
|
|
return f"{_GATE_URL}?{urlencode(params)}"
|
|
|
|
async def _fetch_page_json(
|
|
self,
|
|
rooms: str | None,
|
|
page: int,
|
|
price_min: int | None,
|
|
price_max: int | None,
|
|
new_flat: str = "NO",
|
|
) -> dict[str, Any] | None:
|
|
"""Fetch one gate-API page. Returns parsed JSON dict or None on error/invalid."""
|
|
url = self._build_url(
|
|
page=page, rooms=rooms, price_min=price_min, price_max=price_max, new_flat=new_flat
|
|
)
|
|
try:
|
|
resp = await self._http_get(url, timeout=60)
|
|
except Exception:
|
|
logger.exception("yandex gate: GET failed url=%s", url)
|
|
return None
|
|
if resp.status_code != 200: # type: ignore[union-attr]
|
|
logger.warning("yandex gate: HTTP %d url=%s", resp.status_code, url) # type: ignore[union-attr]
|
|
return None
|
|
try:
|
|
payload: dict[str, Any] = json.loads(resp.text) # type: ignore[union-attr]
|
|
except (json.JSONDecodeError, ValueError):
|
|
logger.warning("yandex gate: JSON parse failed url=%s", url)
|
|
return None
|
|
if _is_gate_error(payload):
|
|
logger.warning("yandex gate: error response url=%s keys=%s", url, list(payload.keys()))
|
|
return None
|
|
return payload
|
|
|
|
async def fetch_around(
|
|
self,
|
|
lat: float,
|
|
lon: float,
|
|
radius_m: int = 1000,
|
|
page: int = 1,
|
|
rooms: str | None = None,
|
|
price_min: int | None = None,
|
|
price_max: int | None = None,
|
|
new_flat: str = "NO",
|
|
) -> list[ScrapedLot]:
|
|
"""Fetch ONE page of gate-API results. lat/lon/radius_m ignored (uses rgid).
|
|
|
|
page: 1-based. Tarpit resilience: status_code==0 or JSON error
|
|
-> rotate IP + retry up to _YANDEX_TARPIT_MAX_RETRIES times.
|
|
Non-tarpit failures (4xx/5xx) are not retried.
|
|
new_flat: "NO" -> vtorichka, "YES" -> novostroyki.
|
|
"""
|
|
url = self._build_url(
|
|
page=page, rooms=rooms, price_min=price_min, price_max=price_max, new_flat=new_flat
|
|
)
|
|
|
|
payload: dict[str, Any] | None = None
|
|
for attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES):
|
|
try:
|
|
response = await self._http_get(url, timeout=60)
|
|
except Exception:
|
|
logger.exception("yandex gate fetch failed: %s", url)
|
|
return []
|
|
|
|
status = response.status_code # type: ignore[union-attr]
|
|
|
|
if status == 0:
|
|
logger.warning(
|
|
"yandex gate: status=0 (tarpit?) rooms=%s page=%d"
|
|
" -- rotating IP + retry attempt %d",
|
|
rooms,
|
|
page,
|
|
attempt + 1,
|
|
)
|
|
await self._rotate_ip()
|
|
await asyncio.sleep(2)
|
|
continue
|
|
|
|
if status != 200:
|
|
logger.warning(
|
|
"yandex gate: HTTP %d rooms=%s page=%d url=%s", status, rooms, page, url
|
|
)
|
|
return []
|
|
|
|
try:
|
|
payload = json.loads(response.text) # type: ignore[union-attr]
|
|
except (json.JSONDecodeError, ValueError):
|
|
logger.warning(
|
|
"yandex gate: JSON parse failed rooms=%s page=%d -- rotating + retry %d",
|
|
rooms,
|
|
page,
|
|
attempt + 1,
|
|
)
|
|
await self._rotate_ip()
|
|
await asyncio.sleep(2)
|
|
continue
|
|
|
|
if _is_gate_error(payload):
|
|
logger.warning(
|
|
"yandex gate: error payload rooms=%s page=%d keys=%s -- rotating + retry %d",
|
|
rooms,
|
|
page,
|
|
list(payload.keys()),
|
|
attempt + 1,
|
|
)
|
|
await self._rotate_ip()
|
|
await asyncio.sleep(2)
|
|
payload = None
|
|
continue
|
|
|
|
break
|
|
|
|
if payload is None:
|
|
logger.warning(
|
|
"yandex gate: all retries exhausted rooms=%s page=%d -- returning empty",
|
|
rooms,
|
|
page,
|
|
)
|
|
return []
|
|
|
|
lots = _parse_gate_json(payload, page_param=page, new_flat=new_flat)
|
|
logger.info(
|
|
"yandex gate page=%d rooms=%s price=[%s-%s] newFlat=%s: %d cards",
|
|
page,
|
|
rooms or "all",
|
|
price_min,
|
|
price_max,
|
|
new_flat,
|
|
len(lots),
|
|
)
|
|
await self.sleep_between_requests()
|
|
return lots
|
|
|
|
async def fetch_around_multi_room(
|
|
self,
|
|
lat: float,
|
|
lon: float,
|
|
radius_m: int = 1000,
|
|
max_pages: int = _GATE_MAX_PAGES_CAP,
|
|
rooms_list: list[str] | None = None,
|
|
price_ranges: list[tuple[int | None, int | None]] | None = None,
|
|
segments: list[str] | None = None,
|
|
**_legacy_kwargs: Any,
|
|
) -> list[ScrapedLot]:
|
|
"""Fetch via segment x rooms x price-range combos; paginate each combo to totalPages.
|
|
|
|
Pagination driven by pager.totalPages from first page response.
|
|
Each combo iterates page=1 .. min(totalPages, max_pages, _GATE_MAX_PAGES_CAP).
|
|
Deduplicates by source_id/source_url across ALL combos AND segments (one `seen`).
|
|
Legacy mode (rooms_list=None, price_ranges=None): single citywide sweep.
|
|
segments: list of newFlat values, default ["NO"] (vtorichka only). Pass
|
|
["NO", "YES"] to sweep both vtorichka and novostroyki in one call.
|
|
"""
|
|
seen: dict[str, ScrapedLot] = {}
|
|
_segments = segments or ["NO"]
|
|
|
|
if rooms_list is None and price_ranges is None:
|
|
combos: list[tuple[str | None, int | None, int | None]] = [(None, None, None)]
|
|
else:
|
|
r_list = rooms_list or list(ROOM_PATH.keys())
|
|
p_ranges = price_ranges or DEFAULT_PRICE_RANGES
|
|
combos = [(r, lo, hi) for r in r_list for lo, hi in p_ranges]
|
|
|
|
for new_flat in _segments:
|
|
_seg = _segment_for(new_flat)
|
|
for rooms, price_min, price_max in combos:
|
|
combo_label = f"{_seg}/{_combo_label(rooms, price_min, price_max)}"
|
|
total_pages: int | None = None
|
|
|
|
for page in range(1, max_pages + 1):
|
|
if page == 1:
|
|
payload_p1: dict[str, Any] | None = None
|
|
p1_url = self._build_url(
|
|
page=1,
|
|
rooms=rooms,
|
|
price_min=price_min,
|
|
price_max=price_max,
|
|
new_flat=new_flat,
|
|
)
|
|
for _attempt in range(1 + _YANDEX_TARPIT_MAX_RETRIES):
|
|
try:
|
|
resp = await self._http_get(p1_url, timeout=60)
|
|
except Exception:
|
|
logger.exception(
|
|
"yandex gate combo fetch failed combo=%s", combo_label
|
|
)
|
|
break
|
|
if resp.status_code == 0: # type: ignore[union-attr]
|
|
logger.warning(
|
|
"yandex gate: tarpit combo=%s page=1 -- rotating", combo_label
|
|
)
|
|
await self._rotate_ip()
|
|
await asyncio.sleep(2)
|
|
continue
|
|
if resp.status_code != 200: # type: ignore[union-attr]
|
|
logger.warning(
|
|
"yandex gate: HTTP %d combo=%s page=1",
|
|
resp.status_code,
|
|
combo_label, # type: ignore[union-attr]
|
|
)
|
|
break
|
|
try:
|
|
payload_p1 = json.loads(resp.text) # type: ignore[union-attr]
|
|
except (json.JSONDecodeError, ValueError):
|
|
logger.warning(
|
|
"yandex gate: JSON parse error combo=%s page=1", combo_label
|
|
)
|
|
payload_p1 = None
|
|
await self._rotate_ip()
|
|
await asyncio.sleep(2)
|
|
continue
|
|
if _is_gate_error(payload_p1):
|
|
logger.warning(
|
|
"yandex gate: error payload combo=%s page=1", combo_label
|
|
)
|
|
payload_p1 = None
|
|
break
|
|
break
|
|
|
|
if payload_p1 is None:
|
|
logger.debug(
|
|
"yandex gate combo [%s] page=1: failed -- skipping", combo_label
|
|
)
|
|
break
|
|
|
|
result = _extract_gate_data(payload_p1)
|
|
if result is None:
|
|
break
|
|
_entities_p1, pager_p1 = result
|
|
total_pages = min(
|
|
pager_p1.get("totalPages", 1),
|
|
max_pages,
|
|
_GATE_MAX_PAGES_CAP,
|
|
)
|
|
lots_p1 = _parse_gate_json(payload_p1, page_param=1, new_flat=new_flat)
|
|
if not lots_p1:
|
|
logger.debug(
|
|
"yandex gate combo [%s] page=1: empty -- stopping", combo_label
|
|
)
|
|
break
|
|
for lot in lots_p1:
|
|
key = lot.source_id or lot.source_url
|
|
if key and key not in seen:
|
|
seen[key] = lot
|
|
await self.sleep_between_requests()
|
|
if total_pages <= 1:
|
|
break
|
|
continue
|
|
|
|
if total_pages is not None and page > total_pages:
|
|
break
|
|
|
|
lots = await self.fetch_around(
|
|
lat,
|
|
lon,
|
|
radius_m,
|
|
page=page,
|
|
rooms=rooms,
|
|
price_min=price_min,
|
|
price_max=price_max,
|
|
new_flat=new_flat,
|
|
)
|
|
if not lots:
|
|
logger.debug(
|
|
"yandex gate combo [%s] page=%d: empty -- stopping",
|
|
combo_label,
|
|
page,
|
|
)
|
|
break
|
|
for lot in lots:
|
|
key = lot.source_id or lot.source_url
|
|
if key and key not in seen:
|
|
seen[key] = lot
|
|
|
|
logger.info(
|
|
"yandex gate aggregate: %d unique lots (%d combos x %d segments=%s)",
|
|
len(seen),
|
|
len(combos),
|
|
len(_segments),
|
|
_segments,
|
|
)
|
|
return list(seen.values())
|
|
|
|
async def fetch_all_secondary(
|
|
self,
|
|
*,
|
|
rooms_buckets: list[str] | None = None,
|
|
price_cap_per_bucket: int = 500,
|
|
max_pages_per_bucket: int = _GATE_MAX_PAGES_CAP,
|
|
concurrency: int = 4,
|
|
on_bucket: Any = None,
|
|
on_progress: Any = None,
|
|
skip_buckets: set[str] | None = None,
|
|
) -> list[ScrapedLot]:
|
|
"""Exhaustive EKB vtorichka load via KOMNATNOST x TSENA partitioning.
|
|
|
|
Uses gate-API JSON (totalItems from pager) instead of DOM state extraction.
|
|
"""
|
|
_buckets = rooms_buckets if rooms_buckets is not None else list(ROOM_PATH.keys())
|
|
seen: dict[str, ScrapedLot] = {}
|
|
|
|
for rooms_key in _buckets:
|
|
logger.info(
|
|
"yandex gate exhaustive: starting rooms=%s (price_cap=%d concurrency=%d)",
|
|
rooms_key,
|
|
price_cap_per_bucket,
|
|
concurrency,
|
|
)
|
|
before = len(seen)
|
|
await self._walk_price_range(
|
|
rooms=rooms_key,
|
|
lo=0,
|
|
hi=_YANDEX_MAX_PRICE,
|
|
seen=seen,
|
|
price_cap_per_bucket=price_cap_per_bucket,
|
|
max_pages_per_bucket=max_pages_per_bucket,
|
|
concurrency=concurrency,
|
|
on_bucket=on_bucket,
|
|
skip_buckets=skip_buckets,
|
|
)
|
|
room_collected = len(seen) - before
|
|
logger.info(
|
|
"yandex gate exhaustive: rooms=%s done -- collected %d (total unique=%d)",
|
|
rooms_key,
|
|
room_collected,
|
|
len(seen),
|
|
)
|
|
if on_progress is not None:
|
|
on_progress(len(seen))
|
|
|
|
logger.info("yandex gate exhaustive: DONE -- total unique=%d lots", len(seen))
|
|
return list(seen.values())
|
|
|
|
async def _walk_price_range(
|
|
self,
|
|
*,
|
|
rooms: str | None,
|
|
lo: int,
|
|
hi: int,
|
|
seen: dict[str, ScrapedLot],
|
|
price_cap_per_bucket: int,
|
|
max_pages_per_bucket: int,
|
|
concurrency: int = 4,
|
|
on_bucket: Any = None,
|
|
skip_buckets: set[str] | None = None,
|
|
_depth: int = 0,
|
|
) -> None:
|
|
"""Recursive binary price-range partitioning.
|
|
|
|
Probe page=1 -> read totalItems from pager.
|
|
totalItems > cap and bracket >= MIN_BRACKET -> split; else paginate leaf.
|
|
Fallback: totalItems=None -> paginate-until-empty (safe degradation).
|
|
skip_buckets: resume checkpoint -- skip leaf buckets already done.
|
|
"""
|
|
_lo_param = lo if lo > 0 else None
|
|
|
|
probe_payload = await self._fetch_page_json(rooms, 1, _lo_param, hi)
|
|
await asyncio.sleep(self.request_delay_sec)
|
|
|
|
total: int | None = None
|
|
probe_lots: list[ScrapedLot] = []
|
|
|
|
if probe_payload is None:
|
|
logger.warning(
|
|
"yandex gate: probe failed rooms=%s [%d, %d] depth=%d -- rotating + retry",
|
|
rooms,
|
|
lo,
|
|
hi,
|
|
_depth,
|
|
)
|
|
rotated = await self._rotate_ip()
|
|
if rotated:
|
|
probe_payload = await self._fetch_page_json(rooms, 1, _lo_param, hi)
|
|
await asyncio.sleep(self.request_delay_sec)
|
|
|
|
if probe_payload is not None:
|
|
result = _extract_gate_data(probe_payload)
|
|
if result is not None:
|
|
_, pager = result
|
|
total = pager.get("totalItems")
|
|
probe_lots = _parse_gate_json(probe_payload, page_param=1)
|
|
else:
|
|
logger.warning(
|
|
"yandex gate: probe data extract failed rooms=%s [%d, %d]", rooms, lo, hi
|
|
)
|
|
|
|
if total is None:
|
|
# Degraded path: paginate-until-empty without knowing total
|
|
logger.warning(
|
|
"yandex gate: totalItems unknown rooms=%s [%d, %d] -- paginate until empty",
|
|
rooms,
|
|
lo,
|
|
hi,
|
|
)
|
|
page = 1
|
|
bucket_key = _combo_label(rooms, lo, hi)
|
|
if skip_buckets and bucket_key in skip_buckets:
|
|
return
|
|
pages_fetched = 0
|
|
while pages_fetched < max_pages_per_bucket:
|
|
payload = await self._fetch_page_json(rooms, page, _lo_param, hi)
|
|
await asyncio.sleep(self.request_delay_sec)
|
|
if payload is None:
|
|
break
|
|
lots = _parse_gate_json(payload, page_param=page)
|
|
if not lots:
|
|
break
|
|
for lot in lots:
|
|
if lot.source_id and lot.source_id not in seen:
|
|
seen[lot.source_id] = lot
|
|
page += 1
|
|
pages_fetched += 1
|
|
if on_bucket is not None:
|
|
on_bucket(bucket_key, len(seen))
|
|
return
|
|
|
|
# Split or paginate
|
|
cap = price_cap_per_bucket
|
|
_min_bracket = 500_000
|
|
bracket = hi - lo
|
|
|
|
if total > cap and bracket >= _min_bracket and _depth < 8:
|
|
mid = (lo + hi) // 2
|
|
await self._walk_price_range(
|
|
rooms=rooms,
|
|
lo=lo,
|
|
hi=mid,
|
|
seen=seen,
|
|
price_cap_per_bucket=cap,
|
|
max_pages_per_bucket=max_pages_per_bucket,
|
|
concurrency=concurrency,
|
|
on_bucket=on_bucket,
|
|
skip_buckets=skip_buckets,
|
|
_depth=_depth + 1,
|
|
)
|
|
await self._walk_price_range(
|
|
rooms=rooms,
|
|
lo=mid + 1,
|
|
hi=hi,
|
|
seen=seen,
|
|
price_cap_per_bucket=cap,
|
|
max_pages_per_bucket=max_pages_per_bucket,
|
|
concurrency=concurrency,
|
|
on_bucket=on_bucket,
|
|
skip_buckets=skip_buckets,
|
|
_depth=_depth + 1,
|
|
)
|
|
return
|
|
|
|
# Leaf bucket: paginate
|
|
bucket_key = _combo_label(rooms, lo, hi)
|
|
if skip_buckets and bucket_key in skip_buckets:
|
|
logger.debug("yandex gate: skip bucket %s (checkpoint)", bucket_key)
|
|
return
|
|
|
|
total_pages = min(
|
|
math.ceil(total / 20),
|
|
_GATE_MAX_PAGES_CAP,
|
|
max_pages_per_bucket,
|
|
)
|
|
logger.info("yandex gate: leaf bucket %s total=%d pages=%d", bucket_key, total, total_pages)
|
|
|
|
# Add probe lots (page 1 already fetched)
|
|
for lot in probe_lots:
|
|
if lot.source_id and lot.source_id not in seen:
|
|
seen[lot.source_id] = lot
|
|
|
|
if total_pages <= 1:
|
|
if on_bucket is not None:
|
|
on_bucket(bucket_key, len(seen))
|
|
return
|
|
|
|
# Paginate pages 2..total_pages with concurrency
|
|
sem = asyncio.Semaphore(concurrency)
|
|
|
|
async def _fetch_leaf_page(pg: int) -> list[ScrapedLot]:
|
|
async with sem:
|
|
payload = await self._fetch_page_json(rooms, pg, _lo_param, hi)
|
|
await asyncio.sleep(self.request_delay_sec)
|
|
if payload is None:
|
|
return []
|
|
return _parse_gate_json(payload, page_param=pg)
|
|
|
|
tasks = [_fetch_leaf_page(pg) for pg in range(2, total_pages + 1)]
|
|
results = await asyncio.gather(*tasks)
|
|
for page_lots in results:
|
|
for lot in page_lots:
|
|
if lot.source_id and lot.source_id not in seen:
|
|
seen[lot.source_id] = lot
|
|
|
|
if on_bucket is not None:
|
|
on_bucket(bucket_key, len(seen))
|
|
|
|
|
|
def _combo_label(rooms: str | None, lo: int | None, hi: int | None) -> str:
|
|
"""Уникальная метка бакета для checkpoint-логики."""
|
|
return f"{rooms or 'any'}:{lo}-{hi}"
|