bot-backend
b9eb478c8f
Route each scraper source (avito/cian/yandex/domclick) to its own camoufox browser+proxy so they no longer wedge each other through a single global egress. Feature-flagged (FEATURE_BROWSER_POOL_ENABLED, default OFF): with the flag off the /fetch and /login paths are byte-for-byte the existing single-browser behavior. When on, /fetch routes by body["source"] to a per-proxy browser via BROWSER_PROXY_MAP (BROWSER_PROXY_AVITO/CIAN/YANDEX/DOMCLICK with legacy fallbacks), each guarded by its own lazy-launched lock. /login stays single-browser in Phase 1. BrowserFetcher gains a source arg (default avito) and sends it in the /fetch body; all scraper callsites pass their source. No docker-compose/.env.runtime changes (Phase 2, owner-gated).
432 lines
16 KiB
Python
432 lines
16 KiB
Python
"""Yandex Realty ЖК landing page parser.
|
||
|
||
URL pattern: /{city}/kupit/novostrojka/<slug>-<id>/
|
||
Reference target: ЖК Татлин (id=1592987, slug=tatlin) — comfort+, June 2023,
|
||
PRINZIP, rating 4.3, 1505 ratings, 353 text reviews, coords (56.855312, 60.576668).
|
||
|
||
Fetch strategy (#974):
|
||
- Все network-запросы (ЖК-лендинг + SERP slug-resolve) идут через BrowserFetcher
|
||
(tradein-browser camoufox), а НЕ через httpx/_http_get.
|
||
- Yandex Realty — JS-heavy / anti-bot; curl_cffi / httpx не получают данные.
|
||
- BrowserFetcher.fetch(url) → str (полный HTML); вызывающий код парсит через parse().
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
import re
|
||
from typing import Any
|
||
|
||
from pydantic import BaseModel, Field
|
||
from selectolax.parser import HTMLParser, Node
|
||
|
||
from app.services.scraper_settings import get_scraper_delay
|
||
from app.services.scrapers.base import BaseScraper
|
||
from app.services.scrapers.yandex_helpers import (
|
||
RE_JK_ID,
|
||
parse_house_class,
|
||
parse_house_type,
|
||
)
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
# ── Models ───────────────────────────────────────────────────────────────────
|
||
|
||
|
||
class JKMetroStation(BaseModel):
|
||
name: str
|
||
walk_min: int | None = None
|
||
|
||
|
||
class YandexNewbuildingInfo(BaseModel):
|
||
"""ЖК landing scrape payload."""
|
||
|
||
ext_id: str # '1592987'
|
||
ext_slug: str # 'tatlin'
|
||
source_url: str
|
||
|
||
# Identification
|
||
name: str | None = None # 'ЖК «Татлин»'
|
||
address: str | None = None # 'Екатеринбург, ул. Черепанова / ул. Готвальда'
|
||
|
||
# Coords (inline in HTML — Yandex unique vs Avito/Cian)
|
||
lat: float | None = None
|
||
lon: float | None = None
|
||
|
||
# Classification + commission
|
||
house_class: str | None = None # 'comfort_plus' etc.
|
||
commission_year: int | None = None
|
||
commission_month: str | None = None # 'июнь' (RU)
|
||
|
||
# Footprint
|
||
total_floors: int | None = None # 35
|
||
corpus_count: int | None = None # 3
|
||
total_area_ha: float | None = None # 1.5
|
||
|
||
# Developer
|
||
developer_name: str | None = None # 'PRINZIP'
|
||
developer_url: str | None = None
|
||
developer_other_jk: list[str] = Field(default_factory=list)
|
||
|
||
# Reviews
|
||
rating: float | None = None # 4.3
|
||
ratings_count: int | None = None # 1505 — оценок
|
||
text_reviews_count: int | None = None # 353 — текстовых отзывов
|
||
|
||
# NLP / description
|
||
description: str | None = None
|
||
|
||
# Metro
|
||
metro_stations: list[JKMetroStation] = Field(default_factory=list)
|
||
|
||
# House type (best-effort NLP from description)
|
||
house_type: str | None = None
|
||
|
||
raw_payload: dict[str, Any] | None = None
|
||
|
||
|
||
# ── Regex constants ───────────────────────────────────────────────────────────
|
||
|
||
RE_FLOORS_TOWERS = re.compile(r"(\d+)-этажн\w+\s+башн", re.IGNORECASE)
|
||
RE_AREA_HA = re.compile(r"участке\s+([\d,]+)\s*га", re.IGNORECASE)
|
||
RE_COMMISSION = re.compile(
|
||
r"(?:введ[её]н|сдан)\s+в\s+эксплуатацию\s+в\s+(\w+)\s+(\d{4})", re.IGNORECASE
|
||
)
|
||
# Yandex drift (#974 follow-up): рейтинг рендерится без пробела перед «из»
|
||
# ("4.3из 5" / "4,5из 5", nbsp внутри) → \s* (а не \s+) перед «из», иначе rating=NULL
|
||
# при заполненных ratings_count/text_reviews (verified live 2026-06-15, ЖК Татлин/Рио).
|
||
RE_RATING = re.compile(r"(\d[.,]\d)\s*из\s+5")
|
||
RE_RATINGS_COUNT = re.compile(r"(\d+)\s+оценок", re.IGNORECASE)
|
||
RE_TEXT_REVIEWS = re.compile(r"Смотреть\s+все\s+(\d+)\s+отзыв", re.IGNORECASE)
|
||
RE_CORPUS_COUNT_WORD = re.compile(
|
||
r"(одн[ау]|две|три|четыре|пять|шесть|семь|восемь|\d+)\s+(?:[\d\s-]*)\s*-?\s*этажн",
|
||
re.IGNORECASE,
|
||
)
|
||
RE_METRO_INLINE = re.compile(
|
||
r"([А-ЯЁ][А-Яа-яё\s-]{2,30}?)\s+(\d+)\s*мин",
|
||
)
|
||
|
||
# Slug из href: /{city}/kupit/novostrojka/<slug>-<id>/
|
||
# Используется в resolve_yandex_jk_slug для извлечения slug из SERP href.
|
||
_JK_SLUG_RE = re.compile(
|
||
r"/(?:ekaterinburg|moskva|spb|[a-z-]+)/kupit/novostrojka/([a-z0-9-]+)-(\d+)/"
|
||
)
|
||
|
||
# Ekb-specific coord ranges (extend per-city later)
|
||
LAT_RANGE = (55.5, 57.5)
|
||
LON_RANGE = (59.5, 61.5)
|
||
|
||
# Word → number map
|
||
_WORD_NUM: dict[str, int] = {
|
||
"одна": 1,
|
||
"одну": 1,
|
||
"одной": 1,
|
||
"две": 2,
|
||
"три": 3,
|
||
"четыре": 4,
|
||
"пять": 5,
|
||
"шесть": 6,
|
||
"семь": 7,
|
||
"восемь": 8,
|
||
}
|
||
|
||
|
||
# ── Scraper class ─────────────────────────────────────────────────────────────
|
||
|
||
|
||
class YandexNewbuildingScraper(BaseScraper):
|
||
name = "yandex_realty_nb"
|
||
base_url = "https://realty.yandex.ru"
|
||
request_delay_sec = 5.0 # class default; instance value loaded from scraper_settings
|
||
|
||
def __init__(self) -> None:
|
||
super().__init__()
|
||
self.request_delay_sec = get_scraper_delay(self.name)
|
||
|
||
async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override]
|
||
raise NotImplementedError(
|
||
"YandexNewbuildingScraper is JK-slug-based; use fetch_jk(slug, id) instead."
|
||
)
|
||
|
||
async def fetch_jk(
|
||
self, jk_slug: str, jk_id: str, city: str = "ekaterinburg"
|
||
) -> YandexNewbuildingInfo | None:
|
||
"""Загрузить ЖК-лендинг через BrowserFetcher и распарсить.
|
||
|
||
Yandex Realty — JS/anti-bot: curl/httpx не получают данные. Запрос идёт
|
||
через tradein-browser (camoufox) контейнер — единственный рабочий путь (#974).
|
||
"""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
url = f"{self.base_url}/{city}/kupit/novostrojka/{jk_slug}-{jk_id}/"
|
||
try:
|
||
async with BrowserFetcher(source="yandex") as fetcher:
|
||
html = await fetcher.fetch(url)
|
||
except Exception:
|
||
logger.exception("yandex nb browser fetch failed: %s", url)
|
||
return None
|
||
if not html or len(html) < 500:
|
||
logger.warning(
|
||
"yandex nb browser returned empty/tiny HTML (%d bytes): %s",
|
||
len(html) if html else 0,
|
||
url,
|
||
)
|
||
return None
|
||
result = self.parse(html, jk_slug=jk_slug, jk_id=jk_id, source_url=url)
|
||
return result
|
||
|
||
def parse(self, html: str, jk_slug: str, jk_id: str, source_url: str) -> YandexNewbuildingInfo:
|
||
tree = HTMLParser(html)
|
||
body = tree.body
|
||
body_text = body.text(strip=True) if body else ""
|
||
|
||
# Header
|
||
h1 = tree.css_first("h1")
|
||
name = h1.text(strip=True) if h1 else None
|
||
|
||
# Sections (heading-based extraction)
|
||
location_text = _find_section_text(tree, "Расположение") or ""
|
||
description_section = (
|
||
_find_section_text(tree, "О комплексе")
|
||
or _find_section_text(tree, "Расположение, транспортная доступность")
|
||
or location_text
|
||
)
|
||
|
||
# Coords — inline scan within plausible range
|
||
lat, lon = _extract_coords(html)
|
||
|
||
# Class
|
||
house_class = parse_house_class(description_section) or parse_house_class(body_text)
|
||
|
||
# Commission
|
||
commission_year: int | None = None
|
||
commission_month: str | None = None
|
||
c_m = RE_COMMISSION.search(description_section) or RE_COMMISSION.search(body_text)
|
||
if c_m:
|
||
commission_month = c_m.group(1).lower()
|
||
try:
|
||
commission_year = int(c_m.group(2))
|
||
except ValueError:
|
||
pass
|
||
|
||
# Floors + corpus + area
|
||
floors_m = RE_FLOORS_TOWERS.search(description_section) or RE_FLOORS_TOWERS.search(
|
||
body_text
|
||
)
|
||
total_floors = int(floors_m.group(1)) if floors_m else None
|
||
|
||
corpus_count = _parse_corpus_count(description_section or body_text)
|
||
|
||
area_m = RE_AREA_HA.search(description_section) or RE_AREA_HA.search(body_text)
|
||
total_area_ha = float(area_m.group(1).replace(",", ".")) if area_m else None
|
||
|
||
# Developer
|
||
dev_link = tree.css_first('[data-test="CARD_DEV_BADGE_DEVELOPER_LINK"]')
|
||
developer_name: str | None = None
|
||
developer_url: str | None = None
|
||
if dev_link is not None:
|
||
developer_name = dev_link.text(strip=True) or None
|
||
href = dev_link.attributes.get("href", "")
|
||
if href:
|
||
developer_url = (
|
||
href if href.startswith("http") else f"https://realty.yandex.ru{href}"
|
||
)
|
||
|
||
# Developer's other JKs
|
||
other_jk_block = tree.css_first('[data-test="CardDevSites"]')
|
||
developer_other_jk: list[str] = []
|
||
if other_jk_block is not None:
|
||
for jk_a in other_jk_block.css("a"):
|
||
t = jk_a.text(strip=True)
|
||
if t and t not in developer_other_jk:
|
||
developer_other_jk.append(t)
|
||
|
||
# Reviews
|
||
rating_m = RE_RATING.search(body_text)
|
||
rating = float(rating_m.group(1).replace(",", ".")) if rating_m else None
|
||
rcount_m = RE_RATINGS_COUNT.search(body_text)
|
||
ratings_count = int(rcount_m.group(1)) if rcount_m else None
|
||
treviews_m = RE_TEXT_REVIEWS.search(body_text)
|
||
text_reviews_count = int(treviews_m.group(1)) if treviews_m else None
|
||
|
||
# Address (header area before lat/lon block)
|
||
address = _extract_jk_address(body_text, name)
|
||
|
||
# Metro stations
|
||
metro_stations = _parse_metro(body_text)
|
||
|
||
# NLP house_type fallback
|
||
house_type = parse_house_type(description_section) or parse_house_type(body_text)
|
||
|
||
return YandexNewbuildingInfo(
|
||
ext_id=jk_id,
|
||
ext_slug=jk_slug,
|
||
source_url=source_url,
|
||
name=name,
|
||
address=address,
|
||
lat=lat,
|
||
lon=lon,
|
||
house_class=house_class,
|
||
commission_year=commission_year,
|
||
commission_month=commission_month,
|
||
total_floors=total_floors,
|
||
corpus_count=corpus_count,
|
||
total_area_ha=total_area_ha,
|
||
developer_name=developer_name,
|
||
developer_url=developer_url,
|
||
developer_other_jk=developer_other_jk[:10],
|
||
rating=rating,
|
||
ratings_count=ratings_count,
|
||
text_reviews_count=text_reviews_count,
|
||
description=description_section or None,
|
||
metro_stations=metro_stations,
|
||
house_type=house_type,
|
||
raw_payload={
|
||
"description_len": len(description_section or ""),
|
||
"body_len": len(body_text),
|
||
},
|
||
)
|
||
|
||
|
||
# ── Slug resolution via SERP ──────────────────────────────────────────────────
|
||
|
||
|
||
async def resolve_yandex_jk_slug(
|
||
jk_id: str,
|
||
city: str = "ekaterinburg",
|
||
) -> str | None:
|
||
"""Найти Yandex Realty slug для ЖК по его ext_id (jk_id) через SERP.
|
||
|
||
Стратегия (#974 — зеркало resolve_cian_zhk_url_via_search):
|
||
1. Запросить поисковую страницу Yandex Realty через BrowserFetcher.
|
||
URL: /ekaterinburg/kupit/novostrojka/?siteId=<jk_id>
|
||
2. В HTML найти первую ссылку вида /<city>/kupit/novostrojka/<slug>-<jk_id>/
|
||
через regex _JK_SLUG_RE.
|
||
3. Вернуть slug или None при любой ошибке.
|
||
|
||
Caller несёт ответственность за anti-bot sleep (зеркало cian_newbuilding.py).
|
||
BrowserFetcher обязателен — Yandex Realty JS/anti-bot, httpx/curl не работают.
|
||
|
||
Returns:
|
||
slug (str без id-суффикса), или None при ошибке / не найден.
|
||
"""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
# Yandex Realty SERP: фильтр по siteId → первый результат = нужный ЖК.
|
||
# Альтернативный путь через Яндекс Поиск (web SERP) менее надёжен из-за
|
||
# вариативности разметки. Прямой realty.yandex.ru SERP — стабильнее.
|
||
serp_url = f"https://realty.yandex.ru/{city}/kupit/novostrojka/?siteId={jk_id}"
|
||
try:
|
||
async with BrowserFetcher(source="yandex") as fetcher:
|
||
html = await fetcher.fetch(serp_url)
|
||
except Exception as exc:
|
||
logger.warning("resolve_yandex_jk_slug jk_id=%s browser fetch failed: %s", jk_id, exc)
|
||
return None
|
||
|
||
if not html:
|
||
logger.warning("resolve_yandex_jk_slug jk_id=%s: empty HTML from browser", jk_id)
|
||
return None
|
||
|
||
# Ищем ссылку вида /{city}/kupit/novostrojka/<slug>-<id>/
|
||
# Ограничиваем: id в ссылке должен совпадать с искомым jk_id.
|
||
for m in _JK_SLUG_RE.finditer(html):
|
||
if m.group(2) == str(jk_id):
|
||
slug = m.group(1)
|
||
logger.info("resolve_yandex_jk_slug jk_id=%s → slug=%s", jk_id, slug)
|
||
return slug
|
||
|
||
logger.warning(
|
||
"resolve_yandex_jk_slug jk_id=%s: no matching slug in SERP HTML (markup drift?)",
|
||
jk_id,
|
||
)
|
||
return None
|
||
|
||
|
||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _extract_coords(html: str) -> tuple[float | None, float | None]:
|
||
"""Scan HTML for any `<lat>.\\d+` and `<lon>.\\d+` matching plausible city range."""
|
||
lats = [
|
||
float(m)
|
||
for m in re.findall(r"\b(\d{2}\.\d{4,8})\b", html)
|
||
if LAT_RANGE[0] <= float(m) <= LAT_RANGE[1]
|
||
]
|
||
lons = [
|
||
float(m)
|
||
for m in re.findall(r"\b(\d{2}\.\d{4,8})\b", html)
|
||
if LON_RANGE[0] <= float(m) <= LON_RANGE[1]
|
||
]
|
||
lat = lats[0] if lats else None
|
||
lon = lons[0] if lons else None
|
||
return lat, lon
|
||
|
||
|
||
def _find_section_text(tree: HTMLParser, heading: str) -> str | None:
|
||
for h in tree.css("h2, h3"):
|
||
if heading.lower() in (h.text(strip=True) or "").lower():
|
||
parts: list[str] = []
|
||
node: Node | None = h.next
|
||
while node is not None:
|
||
tag = (node.tag or "").lower()
|
||
if tag in {"h2", "h3"}:
|
||
break
|
||
txt = node.text(strip=True) if hasattr(node, "text") else ""
|
||
if txt:
|
||
parts.append(txt)
|
||
node = node.next
|
||
return " ".join(parts).strip() or None
|
||
return None
|
||
|
||
|
||
def _parse_corpus_count(text: str) -> int | None:
|
||
m = RE_CORPUS_COUNT_WORD.search(text)
|
||
if not m:
|
||
return None
|
||
token = m.group(1).lower()
|
||
if token.isdigit():
|
||
return int(token)
|
||
return _WORD_NUM.get(token)
|
||
|
||
|
||
def _parse_metro(text: str) -> list[JKMetroStation]:
|
||
stations: list[JKMetroStation] = []
|
||
for m in RE_METRO_INLINE.finditer(text):
|
||
name = m.group(1).strip()
|
||
# filter common noise
|
||
if not 2 <= len(name) <= 40 or "ходьб" in name.lower() or "минут" in name.lower():
|
||
continue
|
||
try:
|
||
stations.append(JKMetroStation(name=name, walk_min=int(m.group(2))))
|
||
except ValueError:
|
||
continue
|
||
if len(stations) >= 5:
|
||
break
|
||
return stations
|
||
|
||
|
||
def _extract_jk_address(body_text: str, name: str | None) -> str | None:
|
||
"""Find address line — typically right after JK name, before metro entries."""
|
||
if name and name in body_text:
|
||
after_name = body_text.split(name, 1)[1]
|
||
# Match 'Город, ул. X / ул. Y' until first metro mention
|
||
m = re.match(
|
||
r"\s*([А-ЯЁ][^•]{5,150}?)(?:\s+[А-ЯЁ][а-яё]+\s+\d+\s*мин|\d+\s+оценок|$)",
|
||
after_name,
|
||
)
|
||
if m:
|
||
addr = m.group(1).strip().rstrip(",").strip()
|
||
if 10 < len(addr) < 200:
|
||
return addr
|
||
return None
|
||
|
||
|
||
# Expose RE_JK_ID at module level for external use (e.g. SERP link → newbuilding detail)
|
||
__all__ = [
|
||
"RE_JK_ID",
|
||
"JKMetroStation",
|
||
"YandexNewbuildingInfo",
|
||
"YandexNewbuildingScraper",
|
||
"resolve_yandex_jk_slug",
|
||
]
|