lekss361
2bcccdf619
Some checks failed
Deploy Trade-In / deploy (push) Blocked by required conditions
Deploy Trade-In / changes (push) Successful in 6s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 31s
Deploy Trade-In / build-backend (push) Has been cancelled
Co-authored-by: lekss361 <lekss361@gendsgn.local> Co-committed-by: lekss361 <lekss361@gendsgn.local>
1050 lines
42 KiB
Python
1050 lines
42 KiB
Python
"""Yandex.Недвижимость scraper (realty.yandex.ru) — DOM-based SERP parser.
|
||
|
||
History: Yandex SSR used to embed full state in `<script id="initial_state_script">`.
|
||
As of 2026-05, the state is chunked + evaporates after hydration. This version
|
||
scrapes the rendered DOM via selectolax + helpers from yandex_helpers.py.
|
||
|
||
Area extraction: DOM concat-text is empty pre-hydration (~82% NULL). Fix: extract
|
||
area from the page-level embedded JSON state (script[id="initial_state_script"] or
|
||
application/json script), keyed by offerId. DOM regex is kept as fallback.
|
||
|
||
Transport (T4): curl_cffi AsyncSession(impersonate="chrome120") — Chrome TLS
|
||
fingerprint bypasses Yandex anti-bot gate that returns captcha/shell HTML for plain
|
||
httpx. Optional cookies loaded from YANDEX_COOKIES_FILE env path.
|
||
|
||
Combos (T5): fetch_around_multi_room supports rooms × price-range combinations to
|
||
work around Yandex SERP ~575-card cap per query. Full combos → ~4000 unique listings.
|
||
|
||
URL pattern (path-based, no geo radius):
|
||
https://realty.yandex.ru/{city}/kupit/kvartira/[{room-slug}/]vtorichniy-rynok/?...
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import inspect
|
||
import json
|
||
import logging
|
||
import math
|
||
import re
|
||
from datetime import date # noqa: F401 (used in type hints / future helpers)
|
||
from pathlib import Path
|
||
from typing import Any
|
||
from urllib.parse import urlencode
|
||
|
||
from curl_cffi.requests import AsyncSession as _CurlCffiSession
|
||
from selectolax.parser import HTMLParser, Node
|
||
|
||
from app.services.scraper_settings import get_scraper_delay
|
||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||
from app.services.scrapers.repair_state_normalizer import infer_repair_state_from_text
|
||
from app.services.scrapers.yandex_helpers import (
|
||
RE_FLOOR,
|
||
RE_JK_ID,
|
||
RE_OFFER_ID,
|
||
RE_PPM2,
|
||
RE_PRICE,
|
||
RE_TITLE_AREA,
|
||
RE_TITLE_ROOMS,
|
||
parse_listing_date,
|
||
parse_rub,
|
||
)
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
DEFAULT_CITY = "ekaterinburg"
|
||
MAX_PAGES = 3
|
||
PHOTO_DOMAIN = "avatars.mds.yandex"
|
||
PHOTO_SIZE_FROM = "app_snippet_small"
|
||
PHOTO_SIZE_TO = "main"
|
||
|
||
# Address container wraps street link + house number text node.
|
||
# Class is hashed but always starts with "AddressWithGeoLinks__addressContainer".
|
||
ADDRESS_CONTAINER_SELECTOR = '[class*="AddressWithGeoLinks__addressContainer"]'
|
||
_RE_WS = re.compile(r"\s+")
|
||
|
||
# Yandex SERP embeds offer data in a <script id="initial_state_script"> as JSON,
|
||
# or (since 2026-05) in <script type="application/json" ...> chunks.
|
||
# We look for these selectors in priority order.
|
||
_STATE_SCRIPT_SELECTORS = [
|
||
'script[id="initial_state_script"]',
|
||
'script[type="application/json"]',
|
||
]
|
||
|
||
# Area field candidates in a Yandex offer object (tried in order).
|
||
_AREA_FIELDS = ("totalArea", "area", "spaceTotal", "spaceAll", "squareTotal")
|
||
|
||
# Captcha detection markers (checked in first 5000 chars, lowercased).
|
||
_CAPTCHA_MARKERS = ("/showcaptcha", "smartcaptcha", "checkboxcaptcha")
|
||
|
||
# Path segments for room buckets (Yandex URL canonical slugs).
|
||
# Key matches the rooms label used in combos; value = URL path segment.
|
||
ROOM_PATH: dict[str, str] = {
|
||
"studio": "studiya",
|
||
"1": "odnokomnatnaya",
|
||
"2": "dvuhkomnatnaya",
|
||
"3": "trehkomnatnaya",
|
||
"4+": "4-i-bolee",
|
||
}
|
||
|
||
# Default price ranges (price_min, price_max) in rubles. None = open-ended.
|
||
DEFAULT_PRICE_RANGES: list[tuple[int | None, int | None]] = [
|
||
(None, 5_000_000),
|
||
(5_000_000, 7_000_000),
|
||
(7_000_000, 10_000_000),
|
||
(10_000_000, 15_000_000),
|
||
(15_000_000, 25_000_000),
|
||
(25_000_000, None),
|
||
]
|
||
|
||
# ── Константы exhaustive-загрузки ──────────────────────────────────────────────
|
||
|
||
# Верхняя граница цены при первом рекурсивном делении (нет явного hi).
|
||
# 200 млн ₽ заведомо выше ТОПа ЕКБ — дальние бакеты дадут 0 офферов.
|
||
_YANDEX_MAX_PRICE = 200_000_000
|
||
|
||
# Минимальный ценовой диапазон для рекурсии; при hi - lo < MIN_BRACKET
|
||
# прекращаем деление и пагинируем как есть.
|
||
_YANDEX_MIN_BRACKET = 50_000
|
||
|
||
# Оценка офферов на страницу Yandex SERP.
|
||
# Yandex показывает ~20-30 карточек на страницу (зависит от типа выдачи).
|
||
# Консервативно 30 — гарантирует, что мы не срежем страницы в leaf-bucket.
|
||
_YANDEX_OFFERS_PER_PAGE = 30
|
||
|
||
# Пути в JSON-state для поиска totalItems (порядок = приоритет).
|
||
_TOTAL_COUNT_PATHS: list[list[str]] = [
|
||
["pageData", "pager", "totalItems"],
|
||
["pageData", "pager", "total"],
|
||
["searchResults", "pager", "totalItems"],
|
||
["pager", "totalItems"],
|
||
["pager", "total"],
|
||
["pageData", "searchQuery", "total"],
|
||
]
|
||
|
||
# Regex для fallback-поиска количества офферов в тексте DOM.
|
||
_RE_TOTAL_COUNT = re.compile(r"(\d[\d ]{0,8})\s+(?:объявлен|квартир|предложен)", re.IGNORECASE)
|
||
|
||
|
||
def _load_cookies_from_file(path: str | None) -> dict[str, str]:
|
||
"""Load Yandex cookies from JSON file path (Netscape/browser export format).
|
||
|
||
Format: [{name: str, value: str, ...}, ...].
|
||
Returns empty dict if path is None, missing, or unparseable — not a failure.
|
||
"""
|
||
if not path:
|
||
return {}
|
||
p = Path(path)
|
||
if not p.exists():
|
||
logger.warning("yandex cookies file not found: %s — running without cookies", path)
|
||
return {}
|
||
try:
|
||
raw = json.loads(p.read_text(encoding="utf-8"))
|
||
result = {
|
||
item["name"]: item["value"]
|
||
for item in raw
|
||
if isinstance(item, dict) and "name" in item and "value" in item
|
||
}
|
||
logger.info("yandex: loaded %d cookies from %s", len(result), path)
|
||
return result
|
||
except Exception:
|
||
logger.warning("yandex: failed to parse cookies file %s — running without", path)
|
||
return {}
|
||
|
||
|
||
def _is_captcha(html: str) -> bool:
|
||
"""Return True if the response looks like a Yandex captcha page."""
|
||
head = html[:5000].lower()
|
||
return any(m in head for m in _CAPTCHA_MARKERS) or "докажите, что вы не робот" in head
|
||
|
||
|
||
def _extract_offer_areas_from_state(html: str) -> dict[str, float]:
|
||
"""Extract {offer_id: area_m2} map from Yandex SERP embedded JSON state.
|
||
|
||
Tries each known script selector, parses JSON, then traverses known
|
||
offer-array paths looking for offer objects with an id + area field.
|
||
|
||
Returns empty dict if state is absent or unparseable.
|
||
"""
|
||
tree = HTMLParser(html)
|
||
candidates: list[str] = []
|
||
for selector in _STATE_SCRIPT_SELECTORS:
|
||
for node in tree.css(selector):
|
||
text = node.text() or ""
|
||
text = text.strip()
|
||
if text:
|
||
candidates.append(text)
|
||
|
||
result: dict[str, float] = {}
|
||
for text in candidates:
|
||
try:
|
||
state = json.loads(text)
|
||
except (json.JSONDecodeError, ValueError):
|
||
continue
|
||
if not isinstance(state, dict):
|
||
continue
|
||
_collect_areas_from_state(state, result)
|
||
if result:
|
||
break # успешно извлекли из первого валидного blob'а
|
||
return result
|
||
|
||
|
||
def _collect_areas_from_state(state: dict[str, Any], out: dict[str, float]) -> None:
|
||
"""Walk known Yandex state paths to find offer objects and extract area.
|
||
|
||
Known paths (tried in order):
|
||
- state["offers"] (flat list)
|
||
- state["pageData"]["offers"] (SERP v2)
|
||
- state["listing"]["offers"] (SERP v3)
|
||
- state["serpData"]["items"] (SERP v4 chunks)
|
||
"""
|
||
offer_lists: list[Any] = []
|
||
for path in (
|
||
["offers"],
|
||
["pageData", "offers"],
|
||
["listing", "offers"],
|
||
["serpData", "items"],
|
||
):
|
||
node: Any = state
|
||
for key in path:
|
||
if not isinstance(node, dict):
|
||
node = None
|
||
break
|
||
node = node.get(key)
|
||
if isinstance(node, list):
|
||
offer_lists.append(node)
|
||
|
||
for offer_list in offer_lists:
|
||
for offer in offer_list:
|
||
if not isinstance(offer, dict):
|
||
continue
|
||
# Flatten one level of nesting (some Yandex shapes wrap the offer)
|
||
inner = offer.get("offer") or offer
|
||
if not isinstance(inner, dict):
|
||
continue
|
||
oid = inner.get("offerId") or inner.get("id")
|
||
if oid is None:
|
||
continue
|
||
area = _read_area_field(inner)
|
||
if area is not None:
|
||
out[str(oid)] = area
|
||
|
||
|
||
def _read_area_field(offer: dict[str, Any]) -> float | None:
|
||
"""Try known field names to read area (м²) from a Yandex offer dict."""
|
||
for field in _AREA_FIELDS:
|
||
val = offer.get(field)
|
||
if val is None:
|
||
continue
|
||
# Some fields are nested dicts: {"value": 52.0, "unit": "SQM"}
|
||
if isinstance(val, dict):
|
||
val = val.get("value")
|
||
if val is not None:
|
||
try:
|
||
result = float(val)
|
||
if result > 0:
|
||
return result
|
||
except (TypeError, ValueError):
|
||
continue
|
||
return None
|
||
|
||
|
||
class YandexRealtyScraper(BaseScraper):
|
||
name = "yandex"
|
||
base_url = "https://realty.yandex.ru"
|
||
request_delay_sec = 5.0
|
||
|
||
def __init__(self, city: str = DEFAULT_CITY) -> None:
|
||
super().__init__()
|
||
self.city = city
|
||
# Load global Yandex delay from DB at instantiation
|
||
self.request_delay_sec = get_scraper_delay(self.name)
|
||
self._cffi_session: _CurlCffiSession | None = None
|
||
self._cookies: dict[str, str] = {}
|
||
|
||
async def __aenter__(self) -> YandexRealtyScraper: # type: ignore[override]
|
||
"""Override: Chrome120 TLS impersonation bypasses Yandex anti-bot gate.
|
||
|
||
Plain httpx returns captcha/shell-HTML on datacenter IPs.
|
||
Sibling pattern: yandex_valuation.py.
|
||
Cookies loaded from YANDEX_COOKIES_FILE settings path (optional).
|
||
"""
|
||
# Deferred import to avoid circular / settings-at-import-time issues
|
||
from app.core.config import settings
|
||
|
||
self._cookies = _load_cookies_from_file(settings.yandex_cookies_file)
|
||
# Mobile proxy wiring (#806 follow-up): Yandex blocks datacenter IPs with
|
||
# captcha/shell HTML even with Chrome TLS fingerprint. Route via mobile proxy.
|
||
# Dedicated YANDEX_PROXY_URL > общий scraper_proxy_url (backward-compat).
|
||
# proxy=None → curl_cffi ходит напрямую (dev без прокси — не падаем).
|
||
_proxy_url = settings.yandex_proxy_url
|
||
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
|
||
self._cffi_session = _CurlCffiSession(
|
||
impersonate="chrome120",
|
||
cookies=self._cookies or {},
|
||
proxies=_proxies,
|
||
headers={
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||
"Accept-Encoding": "gzip, deflate, br",
|
||
"Connection": "keep-alive",
|
||
"Upgrade-Insecure-Requests": "1",
|
||
},
|
||
)
|
||
return self
|
||
|
||
async def __aexit__(self, *args: object) -> None: # type: ignore[override]
|
||
if self._cffi_session is not None:
|
||
await self._cffi_session.close()
|
||
self._cffi_session = None
|
||
|
||
async def _http_get(self, url: str, **kwargs: object) -> object: # type: ignore[override]
|
||
"""curl_cffi-based GET with Chrome120 impersonation.
|
||
|
||
Returns curl_cffi Response (compatible API: .status_code, .text).
|
||
Caller must check status_code; no automatic retry (BaseScraper.retry
|
||
decorator is per-method and not inherited cleanly when overridden).
|
||
"""
|
||
if self._cffi_session is None:
|
||
raise RuntimeError("YandexRealtyScraper must be used as async context manager")
|
||
kwargs.setdefault("timeout", 30)
|
||
return await self._cffi_session.get(url, **kwargs)
|
||
|
||
# ── Anti-block: IP rotation (зеркало cian.py _rotate_ip) ──────────────────
|
||
|
||
async def _rotate_ip(self) -> bool:
|
||
"""Сменить мобильный IP через changeip-ссылку mobileproxy.
|
||
|
||
Дёргается напрямую (без прокси) — это API провайдера, не Yandex. Ждём ~9с:
|
||
мобильному модему нужно время поднять новый IP. Returns True при успехе.
|
||
Зеркало CianScraper._rotate_ip.
|
||
"""
|
||
from app.core.config import settings as _settings
|
||
|
||
rotate_url = _settings.yandex_proxy_rotate_url or _settings.avito_proxy_rotate_url
|
||
if not rotate_url:
|
||
return False
|
||
sep = "&" if "?" in rotate_url else "?"
|
||
try:
|
||
async with _CurlCffiSession(timeout=30) as rot:
|
||
await rot.get(f"{rotate_url}{sep}format=json")
|
||
await asyncio.sleep(9)
|
||
logger.info("yandex proxy: IP rotated via changeip")
|
||
return True
|
||
except Exception:
|
||
logger.warning("yandex proxy: IP rotation failed", exc_info=True)
|
||
return False
|
||
|
||
# ── Total-count extraction (UNVERIFIED — see fallback note) ────────────────
|
||
|
||
def _extract_total_count(self, html: str) -> int | None:
|
||
"""Извлечь totalItems из embedded JSON state Yandex SERP.
|
||
|
||
CAVEAT: Yandex state is chunked + evaporates after hydration (as of 2026-05).
|
||
This is a best-effort extraction — the fallback (paginate-until-empty) is the
|
||
safety net when total is not reliably available. Field is UNVERIFIED against
|
||
live production SERP; paths are taken from known Yandex state shapes.
|
||
|
||
Пути кандидаты (проверяются в порядке приоритета):
|
||
pageData.pager.totalItems
|
||
pageData.pager.total
|
||
searchResults.pager.totalItems
|
||
pager.totalItems
|
||
pager.total
|
||
pageData.searchQuery.total
|
||
"""
|
||
tree = HTMLParser(html)
|
||
candidates: list[str] = []
|
||
for selector in _STATE_SCRIPT_SELECTORS:
|
||
for node in tree.css(selector):
|
||
text = node.text() or ""
|
||
text = text.strip()
|
||
if text:
|
||
candidates.append(text)
|
||
|
||
for blob in candidates:
|
||
try:
|
||
state = json.loads(blob)
|
||
except (json.JSONDecodeError, ValueError):
|
||
continue
|
||
if not isinstance(state, dict):
|
||
continue
|
||
for path in _TOTAL_COUNT_PATHS:
|
||
node_val: Any = state
|
||
for key in path:
|
||
if not isinstance(node_val, dict):
|
||
node_val = None
|
||
break
|
||
node_val = node_val.get(key)
|
||
if node_val is not None:
|
||
try:
|
||
result = int(node_val)
|
||
if result >= 0:
|
||
return result
|
||
except (TypeError, ValueError):
|
||
pass
|
||
|
||
# DOM / regex fallback: try to find a Russian results-count header.
|
||
# Best-effort only — Yandex uses dynamically generated class names.
|
||
# NBSP already normalised to space by caller.
|
||
m = _RE_TOTAL_COUNT.search(html[:20_000])
|
||
if m:
|
||
try:
|
||
return int(m.group(1).replace(" ", ""))
|
||
except (TypeError, ValueError):
|
||
pass
|
||
|
||
return None
|
||
|
||
# ── Leaf-bucket fetch helper ────────────────────────────────────────────────
|
||
|
||
async def _fetch_page_html(
|
||
self,
|
||
rooms: str | None,
|
||
page: int,
|
||
price_min: int | None,
|
||
price_max: int | None,
|
||
) -> str | None:
|
||
"""GET одной SERP-страницы, возвращает HTML или None при ошибке."""
|
||
url = self._build_url(page=page, rooms=rooms, price_min=price_min, price_max=price_max)
|
||
try:
|
||
resp = await self._http_get(url)
|
||
except Exception:
|
||
logger.exception("yandex: GET failed url=%s", url)
|
||
return None
|
||
if resp.status_code != 200: # type: ignore[union-attr]
|
||
logger.warning("yandex: HTTP %d for url=%s", resp.status_code, url) # type: ignore[union-attr]
|
||
return None
|
||
html = resp.text.replace("\xa0", " ") # type: ignore[union-attr]
|
||
return html
|
||
|
||
# ── Exhaustive full load ────────────────────────────────────────────────────
|
||
|
||
async def fetch_all_secondary(
|
||
self,
|
||
*,
|
||
rooms_buckets: list[str] | None = None,
|
||
price_cap_per_bucket: int = 500,
|
||
max_pages_per_bucket: int = 100,
|
||
concurrency: int = 4,
|
||
on_bucket: Any = None,
|
||
on_progress: Any = None,
|
||
) -> list[ScrapedLot]:
|
||
"""Exhaustive-загрузка Yandex ЕКБ вторички через партиционирование КОМНАТНОСТЬ × ЦЕНА.
|
||
|
||
Обходит Yandex SERP-cap (~575 карточек/запрос): внутри каждой комнатности
|
||
адаптивно бьёт ценовой диапазон на бакеты так, чтобы totalItems < price_cap_per_bucket.
|
||
При недоступности totalItems (см. _extract_total_count CAVEAT) переходит на
|
||
paginate-until-empty fallback.
|
||
|
||
Параметры:
|
||
rooms_buckets: список ключей ROOM_PATH для перебора (default: все 5 комнатностей).
|
||
price_cap_per_bucket: максимум офферов в бакете перед делением (< Yandex ~575 cap).
|
||
max_pages_per_bucket: жёсткий cap страниц; 100 × 30 ≈ 3000 > 575.
|
||
concurrency: максимум параллельных page-фетчей в leaf-бакете.
|
||
on_bucket: опциональный callback(list[ScrapedLot]) после каждого leaf-бакета.
|
||
Может быть async или sync. Исключение прерывает прогон.
|
||
on_progress: опциональный callback(unique_count) для heartbeat (per room-bucket).
|
||
|
||
Возвращает list[ScrapedLot] уникальных лотов (дедуп по source_id/source_url).
|
||
"""
|
||
_buckets = rooms_buckets if rooms_buckets is not None else list(ROOM_PATH.keys())
|
||
seen: dict[str, ScrapedLot] = {}
|
||
|
||
for rooms_key in _buckets:
|
||
logger.info(
|
||
"yandex exhaustive: starting rooms=%s (price_cap=%d concurrency=%d)",
|
||
rooms_key,
|
||
price_cap_per_bucket,
|
||
concurrency,
|
||
)
|
||
before = len(seen)
|
||
await self._walk_price_range(
|
||
rooms=rooms_key,
|
||
lo=0,
|
||
hi=_YANDEX_MAX_PRICE,
|
||
seen=seen,
|
||
price_cap_per_bucket=price_cap_per_bucket,
|
||
max_pages_per_bucket=max_pages_per_bucket,
|
||
concurrency=concurrency,
|
||
on_bucket=on_bucket,
|
||
)
|
||
room_collected = len(seen) - before
|
||
logger.info(
|
||
"yandex exhaustive: rooms=%s done — collected %d (total unique=%d)",
|
||
rooms_key,
|
||
room_collected,
|
||
len(seen),
|
||
)
|
||
if on_progress is not None:
|
||
on_progress(len(seen))
|
||
|
||
logger.info("yandex exhaustive: DONE — total unique=%d lots", len(seen))
|
||
return list(seen.values())
|
||
|
||
async def _walk_price_range(
|
||
self,
|
||
*,
|
||
rooms: str | None,
|
||
lo: int,
|
||
hi: int,
|
||
seen: dict[str, ScrapedLot],
|
||
price_cap_per_bucket: int,
|
||
max_pages_per_bucket: int,
|
||
concurrency: int = 4,
|
||
on_bucket: Any = None,
|
||
_depth: int = 0,
|
||
) -> None:
|
||
"""Рекурсивное адаптивное бинарное партиционирование ценового диапазона [lo, hi].
|
||
|
||
Алгоритм:
|
||
1. Запросить page=0 (probe) с price_min=lo, price_max=hi → totalItems.
|
||
2. Если totalItems <= cap → пагинировать leaf-бакет параллельно (concurrency).
|
||
3. Если totalItems > cap → разбить бакет пополам (рекурсия).
|
||
Guard: hi - lo < _YANDEX_MIN_BRACKET → пагинировать как есть.
|
||
Fallback: если totalItems=None → paginate-until-empty (безопасная деградация).
|
||
|
||
# TODO(yandex-total): totalItems not reliably extractable from hydrated DOM
|
||
# — using paginate-until-empty fallback when state extraction fails.
|
||
"""
|
||
_lo_param = lo if lo > 0 else None
|
||
|
||
# ── Шаг 1: probe page=0 ─────────────────────────────────────────────────
|
||
html = await self._fetch_page_html(rooms, 0, _lo_param, hi)
|
||
await asyncio.sleep(self.request_delay_sec)
|
||
|
||
total: int | None = None
|
||
|
||
# CAPTCHA-aware probe: rotate IP + 1 retry
|
||
if html is None or _is_captcha(html):
|
||
logger.warning(
|
||
"yandex: probe captcha/None for rooms=%s [%d, %d] depth=%d — rotating IP + retry",
|
||
rooms,
|
||
lo,
|
||
hi,
|
||
_depth,
|
||
)
|
||
rotated = await self._rotate_ip()
|
||
if rotated:
|
||
html = await self._fetch_page_html(rooms, 0, _lo_param, hi)
|
||
await asyncio.sleep(self.request_delay_sec)
|
||
|
||
if html is not None and not _is_captcha(html):
|
||
total = self._extract_total_count(html)
|
||
|
||
# ── Fallback: totalItems not available ──────────────────────────────────
|
||
# TODO(yandex-total): totalItems not reliably extractable from hydrated DOM
|
||
# — using paginate-until-empty fallback when state extraction fails.
|
||
if total is None:
|
||
logger.warning(
|
||
"yandex: totalItems=None for rooms=%s [%d, %d] depth=%d "
|
||
"— using paginate-until-empty fallback",
|
||
rooms,
|
||
lo,
|
||
hi,
|
||
_depth,
|
||
)
|
||
bucket_lots: list[ScrapedLot] = []
|
||
# page=0 probe already fetched — reuse its HTML if available
|
||
if html is not None and not _is_captcha(html):
|
||
probe_lots = self._parse_html(html, page=0)
|
||
bucket_lots.extend(probe_lots)
|
||
start_page = 1
|
||
else:
|
||
start_page = 0
|
||
|
||
for p in range(start_page, max_pages_per_bucket):
|
||
if p == 0:
|
||
page_lots: list[ScrapedLot] = []
|
||
else:
|
||
page_html = await self._fetch_page_html(rooms, p, _lo_param, hi)
|
||
await asyncio.sleep(self.request_delay_sec)
|
||
if page_html is None or _is_captcha(page_html):
|
||
logger.warning(
|
||
"yandex: fallback page=%d captcha/None rooms=%s [%d, %d] — stopping",
|
||
p,
|
||
rooms,
|
||
lo,
|
||
hi,
|
||
)
|
||
break
|
||
page_lots = self._parse_html(page_html, page=p)
|
||
if not page_lots and p > start_page:
|
||
break
|
||
bucket_lots.extend(page_lots)
|
||
|
||
for lot in bucket_lots:
|
||
key = lot.source_id or lot.source_url
|
||
if key:
|
||
seen[key] = lot
|
||
|
||
if on_bucket and bucket_lots:
|
||
res = on_bucket(bucket_lots)
|
||
if inspect.isawaitable(res):
|
||
await res
|
||
return
|
||
|
||
logger.info(
|
||
"yandex: rooms=%s [%d, %d] totalItems=%d depth=%d",
|
||
rooms,
|
||
lo,
|
||
hi,
|
||
total,
|
||
_depth,
|
||
)
|
||
|
||
if total == 0:
|
||
return
|
||
|
||
# ── Шаг 2: деление или пагинация ────────────────────────────────────────
|
||
bracket_size = hi - lo
|
||
need_split = total > price_cap_per_bucket
|
||
too_narrow = bracket_size < _YANDEX_MIN_BRACKET
|
||
|
||
if need_split and too_narrow:
|
||
logger.warning(
|
||
"yandex: rooms=%s [%d, %d] totalItems=%d > cap=%d but bracket=%d < "
|
||
"MIN_BRACKET=%d — paginating as-is (tail loss ~%d)",
|
||
rooms,
|
||
lo,
|
||
hi,
|
||
total,
|
||
price_cap_per_bucket,
|
||
bracket_size,
|
||
_YANDEX_MIN_BRACKET,
|
||
max(0, total - price_cap_per_bucket),
|
||
)
|
||
need_split = False
|
||
|
||
if need_split:
|
||
mid = (lo + hi) // 2
|
||
await self._walk_price_range(
|
||
rooms=rooms,
|
||
lo=lo,
|
||
hi=mid,
|
||
seen=seen,
|
||
price_cap_per_bucket=price_cap_per_bucket,
|
||
max_pages_per_bucket=max_pages_per_bucket,
|
||
concurrency=concurrency,
|
||
on_bucket=on_bucket,
|
||
_depth=_depth + 1,
|
||
)
|
||
await self._walk_price_range(
|
||
rooms=rooms,
|
||
lo=mid + 1,
|
||
hi=hi,
|
||
seen=seen,
|
||
price_cap_per_bucket=price_cap_per_bucket,
|
||
max_pages_per_bucket=max_pages_per_bucket,
|
||
concurrency=concurrency,
|
||
on_bucket=on_bucket,
|
||
_depth=_depth + 1,
|
||
)
|
||
return
|
||
|
||
# ── Параллельная пагинация leaf-бакета ─────────────────────────────────
|
||
# Yandex SERP pages are 0-indexed (unlike Cian which is 1-indexed)
|
||
max_pages = min(
|
||
math.ceil(total / _YANDEX_OFFERS_PER_PAGE),
|
||
max_pages_per_bucket,
|
||
)
|
||
|
||
sem = asyncio.Semaphore(concurrency)
|
||
|
||
async def _one_page(p: int) -> list[ScrapedLot]:
|
||
if p == 0:
|
||
# Reuse probe HTML
|
||
return self._parse_html(html, page=0) if html else []
|
||
async with sem:
|
||
page_html = await self._fetch_page_html(rooms, p, _lo_param, hi)
|
||
await asyncio.sleep(self.request_delay_sec)
|
||
if page_html is None or _is_captcha(page_html):
|
||
logger.warning(
|
||
"yandex: page_html=None/captcha rooms=%s [%d, %d] page=%d — skipping",
|
||
rooms,
|
||
lo,
|
||
hi,
|
||
p,
|
||
)
|
||
return []
|
||
return self._parse_html(page_html, page=p)
|
||
|
||
page_results = await asyncio.gather(
|
||
*[_one_page(p) for p in range(0, max_pages)],
|
||
return_exceptions=True,
|
||
)
|
||
|
||
bucket_lots_leaf: list[ScrapedLot] = []
|
||
for p_idx, res in enumerate(page_results):
|
||
if isinstance(res, BaseException):
|
||
logger.warning(
|
||
"yandex: page exception rooms=%s [%d, %d] page=%d — %r",
|
||
rooms,
|
||
lo,
|
||
hi,
|
||
p_idx,
|
||
res,
|
||
)
|
||
else:
|
||
bucket_lots_leaf.extend(res)
|
||
|
||
for lot in bucket_lots_leaf:
|
||
key = lot.source_id or lot.source_url
|
||
if key:
|
||
seen[key] = lot
|
||
|
||
logger.info(
|
||
"yandex: rooms=%s [%d, %d] paginated=%d pages collected=%d unique_total=%d",
|
||
rooms,
|
||
lo,
|
||
hi,
|
||
max_pages,
|
||
len(bucket_lots_leaf),
|
||
len(seen),
|
||
)
|
||
|
||
if on_bucket and bucket_lots_leaf:
|
||
res_cb = on_bucket(bucket_lots_leaf)
|
||
if inspect.isawaitable(res_cb):
|
||
await res_cb
|
||
|
||
async def fetch_around(
|
||
self,
|
||
lat: float,
|
||
lon: float,
|
||
radius_m: int = 1000,
|
||
page: int = 0,
|
||
rooms: str | None = None,
|
||
price_min: int | None = None,
|
||
price_max: int | None = None,
|
||
) -> list[ScrapedLot]:
|
||
"""Fetch ONE page of SERP cards.
|
||
|
||
lat/lon/radius_m ignored (Yandex uses path-based vtorichka URL);
|
||
kept for BaseScraper compat + logging.
|
||
rooms: room-bucket slug (one of ROOM_PATH keys) or None = all rooms.
|
||
price_min/price_max: price filters in rubles (None = open-ended).
|
||
"""
|
||
url = self._build_url(page=page, rooms=rooms, price_min=price_min, price_max=price_max)
|
||
try:
|
||
response = await self._http_get(url)
|
||
except Exception:
|
||
logger.exception("yandex serp fetch failed: %s", url)
|
||
return []
|
||
if response.status_code != 200:
|
||
logger.warning("yandex serp returned %d for %s", response.status_code, url)
|
||
return []
|
||
|
||
# NBSP → space: parse_rub() buggy on \xa0 in capture groups → price=None (T4)
|
||
html = response.text.replace("\xa0", " ")
|
||
|
||
if _is_captcha(html):
|
||
logger.warning(
|
||
"yandex serp: captcha detected on page=%d url=%s — returning empty",
|
||
page,
|
||
url,
|
||
)
|
||
return []
|
||
|
||
lots = self._parse_html(html, page=page)
|
||
logger.info(
|
||
"yandex serp page=%d city=%s rooms=%s price=[%s-%s]: %d cards",
|
||
page,
|
||
self.city,
|
||
rooms or "all",
|
||
price_min,
|
||
price_max,
|
||
len(lots),
|
||
)
|
||
await self.sleep_between_requests()
|
||
return lots
|
||
|
||
async def fetch_around_multi_room(
|
||
self,
|
||
lat: float,
|
||
lon: float,
|
||
radius_m: int = 1000,
|
||
max_pages: int = MAX_PAGES,
|
||
rooms_list: list[str] | None = None,
|
||
price_ranges: list[tuple[int | None, int | None]] | None = None,
|
||
**_legacy_kwargs: Any, # swallow old callers' kwargs
|
||
) -> list[ScrapedLot]:
|
||
"""Fetch SERP via rooms × price-range combos to bypass SERP ~575-card cap.
|
||
|
||
When rooms_list/price_ranges are provided (T5 combos mode), iterates all
|
||
(room, price_min, price_max) combinations, each up to max_pages pages,
|
||
deduplicating by source_id/source_url across combos.
|
||
|
||
When both are None (legacy mode), falls back to single citywide sweep
|
||
(original behaviour — up to max_pages pages, no filtering).
|
||
"""
|
||
seen: dict[str, ScrapedLot] = {}
|
||
|
||
if rooms_list is None and price_ranges is None:
|
||
# Legacy / citywide mode — no combos
|
||
combos: list[tuple[str | None, int | None, int | None]] = [(None, None, None)]
|
||
else:
|
||
r_list = rooms_list or list(ROOM_PATH.keys())
|
||
p_ranges = price_ranges or DEFAULT_PRICE_RANGES
|
||
combos = [(r, lo, hi) for r in r_list for lo, hi in p_ranges]
|
||
|
||
for rooms, price_min, price_max in combos:
|
||
combo_label = _combo_label(rooms, price_min, price_max)
|
||
for page in range(max_pages):
|
||
lots = await self.fetch_around(
|
||
lat,
|
||
lon,
|
||
radius_m,
|
||
page=page,
|
||
rooms=rooms,
|
||
price_min=price_min,
|
||
price_max=price_max,
|
||
)
|
||
if not lots:
|
||
logger.debug(
|
||
"yandex combos [%s] page=%d: empty — stopping combo",
|
||
combo_label,
|
||
page,
|
||
)
|
||
break
|
||
for lot in lots:
|
||
key = lot.source_id or lot.source_url
|
||
if key and key not in seen:
|
||
seen[key] = lot
|
||
|
||
logger.info(
|
||
"yandex serp aggregate: %d unique lots (city=%s, %d combos)",
|
||
len(seen),
|
||
self.city,
|
||
len(combos),
|
||
)
|
||
return list(seen.values())
|
||
|
||
def _build_url(
|
||
self,
|
||
page: int = 0,
|
||
rooms: str | None = None,
|
||
price_min: int | None = None,
|
||
price_max: int | None = None,
|
||
) -> str:
|
||
"""Build Yandex SERP URL.
|
||
|
||
rooms: key from ROOM_PATH for path-based room bucket; None = all rooms.
|
||
price_min/price_max: optional price filters in rubles.
|
||
newFlat=NO_DEAL is added when any price filter is set — prevents Yandex
|
||
from canonicalizing away /vtorichniy-rynok/ for some room slugs.
|
||
"""
|
||
if rooms and rooms in ROOM_PATH:
|
||
path = f"/{self.city}/kupit/kvartira/{ROOM_PATH[rooms]}/vtorichniy-rynok/"
|
||
else:
|
||
path = f"/{self.city}/kupit/kvartira/vtorichniy-rynok/"
|
||
base = f"{self.base_url}{path}"
|
||
|
||
params: dict[str, str | int] = {}
|
||
if price_min is not None:
|
||
params["priceMin"] = price_min
|
||
if price_max is not None:
|
||
params["priceMax"] = price_max
|
||
if price_min is not None or price_max is not None:
|
||
# Guarantee vtorichka — Yandex sometimes drops /vtorichniy-rynok/ on canonicalize
|
||
params["newFlat"] = "NO_DEAL"
|
||
if page > 0:
|
||
params["page"] = page
|
||
|
||
if params:
|
||
return f"{base}?{urlencode(params)}"
|
||
return base
|
||
|
||
def _parse_html(self, html: str, page: int = 0) -> list[ScrapedLot]:
|
||
# NBSP-fix applied by caller (fetch_around) before _parse_html;
|
||
# also applied here as a safety net for direct callers (tests, scripts).
|
||
html = html.replace("\xa0", " ")
|
||
|
||
# Extract offer-level area from embedded JSON state first (covers ~82% of
|
||
# cards where DOM text is empty pre-hydration). Falls back to DOM regex per card.
|
||
state_areas = _extract_offer_areas_from_state(html)
|
||
if state_areas:
|
||
logger.debug("yandex serp: state areas extracted for %d offers", len(state_areas))
|
||
tree = HTMLParser(html)
|
||
cards = tree.css('[data-test="OffersSerpItem"]')
|
||
lots: list[ScrapedLot] = []
|
||
for card in cards:
|
||
lot = self._card_to_lot(card, page=page, state_areas=state_areas)
|
||
if lot is not None:
|
||
lots.append(lot)
|
||
return lots
|
||
|
||
def _card_to_lot(
|
||
self,
|
||
card: Node,
|
||
page: int = 0,
|
||
state_areas: dict[str, float] | None = None,
|
||
) -> ScrapedLot | None:
|
||
try:
|
||
# offer_id — required
|
||
offer_link = card.css_first('a[href^="/offer/"]')
|
||
if offer_link is None:
|
||
return None
|
||
href = offer_link.attributes.get("href", "")
|
||
id_match = RE_OFFER_ID.search(href)
|
||
if not id_match:
|
||
return None
|
||
offer_id = id_match.group(1)
|
||
|
||
text = card.text(strip=True)
|
||
|
||
# Price — required (ScrapedLot.price_rub > 0)
|
||
price_m = RE_PRICE.search(text)
|
||
price_rub = parse_rub(price_m.group(1)) if price_m else None
|
||
if not price_rub or price_rub <= 0:
|
||
return None
|
||
|
||
# Area: prefer JSON state (covers pre-hydration DOM where text is empty),
|
||
# fall back to DOM regex.
|
||
area_m2: float | None = (state_areas or {}).get(offer_id)
|
||
if area_m2 is None:
|
||
area_m = RE_TITLE_AREA.search(text)
|
||
area_m2 = float(area_m.group(1).replace(",", ".")) if area_m else None
|
||
|
||
rooms = self._parse_rooms(text)
|
||
|
||
floor_m = RE_FLOOR.search(text)
|
||
floor = int(floor_m.group(1)) if floor_m else None
|
||
total_floors = int(floor_m.group(2)) if floor_m else None
|
||
|
||
ppm2_m = RE_PPM2.search(text)
|
||
price_per_m2 = parse_rub(ppm2_m.group(1)) if ppm2_m else None
|
||
|
||
bargain = "торг" in text.lower()
|
||
listing_date = parse_listing_date(text)
|
||
|
||
# Repair state — инференс из текста карточки (#622). Yandex SERP не
|
||
# отдаёт структурного поля ремонта; берём сигнал из текста сниппета.
|
||
repair_state = infer_repair_state_from_text(text)
|
||
|
||
# Address — prefer the full address container (street + house number);
|
||
# fall back to the street aggregator link (street name only) if absent.
|
||
# See _extract_address for details.
|
||
address = self._extract_address(card)
|
||
|
||
# Photos
|
||
photo_urls = self._extract_photos(card)
|
||
|
||
# Newbuilding linkage
|
||
house_source: str | None = None
|
||
house_ext_id: str | None = None
|
||
house_url: str | None = None
|
||
nb_link = card.css_first('a[href*="/kupit/novostrojka/"]')
|
||
if nb_link is not None:
|
||
nb_href = nb_link.attributes.get("href", "")
|
||
nb_match = RE_JK_ID.search(nb_href)
|
||
if nb_match:
|
||
house_source = "yandex_realty_nb"
|
||
house_ext_id = nb_match.group(2)
|
||
house_url = (
|
||
nb_href
|
||
if nb_href.startswith("http")
|
||
else f"https://realty.yandex.ru{nb_href}"
|
||
)
|
||
|
||
return ScrapedLot(
|
||
source=self.name,
|
||
source_url=f"https://realty.yandex.ru/offer/{offer_id}/",
|
||
source_id=offer_id,
|
||
address=address,
|
||
lat=None,
|
||
lon=None,
|
||
rooms=rooms,
|
||
area_m2=area_m2,
|
||
floor=floor,
|
||
total_floors=total_floors,
|
||
repair_state=repair_state,
|
||
price_rub=price_rub,
|
||
price_per_m2=price_per_m2,
|
||
bargain_allowed=bargain,
|
||
listing_date=listing_date,
|
||
photo_urls=photo_urls,
|
||
house_source=house_source,
|
||
house_ext_id=house_ext_id,
|
||
house_url=house_url,
|
||
listing_segment="vtorichka",
|
||
raw_payload={
|
||
"card_text": text[:1000],
|
||
"page": page,
|
||
"city": self.city,
|
||
},
|
||
)
|
||
except Exception:
|
||
logger.exception("yandex card parse failed (page=%d)", page)
|
||
return None
|
||
|
||
@staticmethod
|
||
def _extract_address(card: Node) -> str | None:
|
||
"""Extract full address (street + house number) from a SERP card.
|
||
|
||
Yandex renders the address inside a container like:
|
||
<div class="AddressWithGeoLinks__addressContainer--XXXX">
|
||
<a href="/.../kupit/kvartira/st-...">улица Энгельса</a>, 38
|
||
</div>
|
||
|
||
The street-only `<a>` tag was used historically — it gave just the
|
||
street name and broke forward-geocoding precision (street-level only,
|
||
not exact house). Reading the container's text yields the full
|
||
"<street>, <house>" pair, including letter/fraction suffixes
|
||
(e.g. "2В", "2/2", "14к2"). Some cards prefix district/city info
|
||
(e.g. "Берёзовский, Александровский проспект, 5А") — that's still
|
||
a valid, more specific address for the geocoder.
|
||
|
||
Falls back to the street-only link when the container is missing
|
||
(older layout / unexpected DOM) so we never regress to NULL.
|
||
"""
|
||
addr_div = card.css_first(ADDRESS_CONTAINER_SELECTOR)
|
||
if addr_div is not None:
|
||
# text(strip=False) preserves spaces between text nodes and
|
||
# the anchor's text (selectolax with strip=True collapses them
|
||
# and yields "улица Энгельса,38" instead of "улица Энгельса, 38").
|
||
raw = addr_div.text(strip=False) or ""
|
||
normalized = _RE_WS.sub(" ", raw).strip().strip(",").strip()
|
||
if normalized:
|
||
return normalized
|
||
street_link = card.css_first('a[href*="/kupit/kvartira/st-"]')
|
||
return street_link.text(strip=True) if street_link else None
|
||
|
||
@staticmethod
|
||
def _parse_rooms(text: str) -> int | None:
|
||
m = RE_TITLE_ROOMS.search(text)
|
||
if not m:
|
||
return None
|
||
if m.group(2): # studio
|
||
return 0
|
||
if m.group(1): # numbered
|
||
try:
|
||
return int(m.group(1))
|
||
except ValueError:
|
||
return None
|
||
return None
|
||
|
||
@staticmethod
|
||
def _extract_photos(card: Node) -> list[str]:
|
||
urls: list[str] = []
|
||
for img in card.css("img"):
|
||
src = img.attributes.get("src", "") or ""
|
||
if PHOTO_DOMAIN in src:
|
||
urls.append(src.replace(PHOTO_SIZE_FROM, PHOTO_SIZE_TO))
|
||
if len(urls) >= 5:
|
||
break
|
||
return urls
|
||
|
||
|
||
def _combo_label(
|
||
rooms: str | None,
|
||
price_min: int | None,
|
||
price_max: int | None,
|
||
) -> str:
|
||
"""Human-readable label for a (rooms, price_min, price_max) combo."""
|
||
r = rooms or "all-rooms"
|
||
if price_min is None and price_max is None:
|
||
return f"{r}/any-price"
|
||
lo = f"{price_min // 1_000_000}M" if price_min is not None else "0"
|
||
hi = f"{price_max // 1_000_000}M" if price_max is not None else "inf"
|
||
return f"{r}/{lo}-{hi}"
|