"""Yandex Realty ЖК landing page parser. URL pattern: /{city}/kupit/novostrojka/-/ Reference target: ЖК Татлин (id=1592987, slug=tatlin) — comfort+, June 2023, PRINZIP, rating 4.3, 1505 ratings, 353 text reviews, coords (56.855312, 60.576668). """ from __future__ import annotations import logging import re from typing import Any from pydantic import BaseModel, Field from selectolax.parser import HTMLParser, Node from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper from app.services.scrapers.yandex_helpers import ( RE_JK_ID, parse_house_class, parse_house_type, ) logger = logging.getLogger(__name__) # ── Models ─────────────────────────────────────────────────────────────────── class JKMetroStation(BaseModel): name: str walk_min: int | None = None class YandexNewbuildingInfo(BaseModel): """ЖК landing scrape payload.""" ext_id: str # '1592987' ext_slug: str # 'tatlin' source_url: str # Identification name: str | None = None # 'ЖК «Татлин»' address: str | None = None # 'Екатеринбург, ул. Черепанова / ул. Готвальда' # Coords (inline in HTML — Yandex unique vs Avito/Cian) lat: float | None = None lon: float | None = None # Classification + commission house_class: str | None = None # 'comfort_plus' etc. commission_year: int | None = None commission_month: str | None = None # 'июнь' (RU) # Footprint total_floors: int | None = None # 35 corpus_count: int | None = None # 3 total_area_ha: float | None = None # 1.5 # Developer developer_name: str | None = None # 'PRINZIP' developer_url: str | None = None developer_other_jk: list[str] = Field(default_factory=list) # Reviews rating: float | None = None # 4.3 ratings_count: int | None = None # 1505 — оценок text_reviews_count: int | None = None # 353 — текстовых отзывов # NLP / description description: str | None = None # Metro metro_stations: list[JKMetroStation] = Field(default_factory=list) # House type (best-effort NLP from description) house_type: str | None = None raw_payload: dict[str, Any] | None = None # ── Regex constants ─────────────────────────────────────────────────────────── RE_FLOORS_TOWERS = re.compile(r"(\d+)-этажн\w+\s+башн", re.IGNORECASE) RE_AREA_HA = re.compile(r"участке\s+([\d,]+)\s*га", re.IGNORECASE) RE_COMMISSION = re.compile( r"(?:введ[её]н|сдан)\s+в\s+эксплуатацию\s+в\s+(\w+)\s+(\d{4})", re.IGNORECASE ) RE_RATING = re.compile(r"(\d[.,]\d)\s+из\s+5") RE_RATINGS_COUNT = re.compile(r"(\d+)\s+оценок", re.IGNORECASE) RE_TEXT_REVIEWS = re.compile(r"Смотреть\s+все\s+(\d+)\s+отзыв", re.IGNORECASE) RE_CORPUS_COUNT_WORD = re.compile( r"(одн[ау]|две|три|четыре|пять|шесть|семь|восемь|\d+)\s+(?:[\d\s-]*)\s*-?\s*этажн", re.IGNORECASE, ) RE_METRO_INLINE = re.compile( r"([А-ЯЁ][А-Яа-яё\s-]{2,30}?)\s+(\d+)\s*мин", ) # Ekb-specific coord ranges (extend per-city later) LAT_RANGE = (55.5, 57.5) LON_RANGE = (59.5, 61.5) # Word → number map _WORD_NUM: dict[str, int] = { "одна": 1, "одну": 1, "одной": 1, "две": 2, "три": 3, "четыре": 4, "пять": 5, "шесть": 6, "семь": 7, "восемь": 8, } # ── Scraper class ───────────────────────────────────────────────────────────── class YandexNewbuildingScraper(BaseScraper): name = "yandex_realty_nb" base_url = "https://realty.yandex.ru" request_delay_sec = 5.0 # class default; instance value loaded from scraper_settings def __init__(self) -> None: super().__init__() self.request_delay_sec = get_scraper_delay(self.name) async def fetch_around( self, lat: float, lon: float, radius_m: int = 1000 ) -> list: # type: ignore[override] raise NotImplementedError( "YandexNewbuildingScraper is JK-slug-based; use fetch_jk(slug, id) instead." ) async def fetch_jk( self, jk_slug: str, jk_id: str, city: str = "ekaterinburg" ) -> YandexNewbuildingInfo | None: url = f"{self.base_url}/{city}/kupit/novostrojka/{jk_slug}-{jk_id}/" try: response = await self._http_get(url) except Exception: logger.exception("yandex nb fetch failed: %s", url) return None if response.status_code != 200: logger.warning("yandex nb returned %d for %s", response.status_code, url) return None result = self.parse(response.text, jk_slug=jk_slug, jk_id=jk_id, source_url=url) await self.sleep_between_requests() return result def parse( self, html: str, jk_slug: str, jk_id: str, source_url: str ) -> YandexNewbuildingInfo: tree = HTMLParser(html) body = tree.body body_text = body.text(strip=True) if body else "" # Header h1 = tree.css_first("h1") name = h1.text(strip=True) if h1 else None # Sections (heading-based extraction) location_text = _find_section_text(tree, "Расположение") or "" description_section = ( _find_section_text(tree, "О комплексе") or _find_section_text(tree, "Расположение, транспортная доступность") or location_text ) # Coords — inline scan within plausible range lat, lon = _extract_coords(html) # Class house_class = parse_house_class(description_section) or parse_house_class(body_text) # Commission commission_year: int | None = None commission_month: str | None = None c_m = RE_COMMISSION.search(description_section) or RE_COMMISSION.search(body_text) if c_m: commission_month = c_m.group(1).lower() try: commission_year = int(c_m.group(2)) except ValueError: pass # Floors + corpus + area floors_m = RE_FLOORS_TOWERS.search(description_section) or RE_FLOORS_TOWERS.search( body_text ) total_floors = int(floors_m.group(1)) if floors_m else None corpus_count = _parse_corpus_count(description_section or body_text) area_m = RE_AREA_HA.search(description_section) or RE_AREA_HA.search(body_text) total_area_ha = float(area_m.group(1).replace(",", ".")) if area_m else None # Developer dev_link = tree.css_first('[data-test="CARD_DEV_BADGE_DEVELOPER_LINK"]') developer_name: str | None = None developer_url: str | None = None if dev_link is not None: developer_name = dev_link.text(strip=True) or None href = dev_link.attributes.get("href", "") if href: developer_url = ( href if href.startswith("http") else f"https://realty.yandex.ru{href}" ) # Developer's other JKs other_jk_block = tree.css_first('[data-test="CardDevSites"]') developer_other_jk: list[str] = [] if other_jk_block is not None: for jk_a in other_jk_block.css("a"): t = jk_a.text(strip=True) if t and t not in developer_other_jk: developer_other_jk.append(t) # Reviews rating_m = RE_RATING.search(body_text) rating = float(rating_m.group(1).replace(",", ".")) if rating_m else None rcount_m = RE_RATINGS_COUNT.search(body_text) ratings_count = int(rcount_m.group(1)) if rcount_m else None treviews_m = RE_TEXT_REVIEWS.search(body_text) text_reviews_count = int(treviews_m.group(1)) if treviews_m else None # Address (header area before lat/lon block) address = _extract_jk_address(body_text, name) # Metro stations metro_stations = _parse_metro(body_text) # NLP house_type fallback house_type = parse_house_type(description_section) or parse_house_type(body_text) return YandexNewbuildingInfo( ext_id=jk_id, ext_slug=jk_slug, source_url=source_url, name=name, address=address, lat=lat, lon=lon, house_class=house_class, commission_year=commission_year, commission_month=commission_month, total_floors=total_floors, corpus_count=corpus_count, total_area_ha=total_area_ha, developer_name=developer_name, developer_url=developer_url, developer_other_jk=developer_other_jk[:10], rating=rating, ratings_count=ratings_count, text_reviews_count=text_reviews_count, description=description_section or None, metro_stations=metro_stations, house_type=house_type, raw_payload={ "description_len": len(description_section or ""), "body_len": len(body_text), }, ) # ── helpers ─────────────────────────────────────────────────────────────────── def _extract_coords(html: str) -> tuple[float | None, float | None]: """Scan HTML for any `.\\d+` and `.\\d+` matching plausible city range.""" lats = [ float(m) for m in re.findall(r"\b(\d{2}\.\d{4,8})\b", html) if LAT_RANGE[0] <= float(m) <= LAT_RANGE[1] ] lons = [ float(m) for m in re.findall(r"\b(\d{2}\.\d{4,8})\b", html) if LON_RANGE[0] <= float(m) <= LON_RANGE[1] ] lat = lats[0] if lats else None lon = lons[0] if lons else None return lat, lon def _find_section_text(tree: HTMLParser, heading: str) -> str | None: for h in tree.css("h2, h3"): if heading.lower() in (h.text(strip=True) or "").lower(): parts: list[str] = [] node: Node | None = h.next while node is not None: tag = (node.tag or "").lower() if tag in {"h2", "h3"}: break txt = node.text(strip=True) if hasattr(node, "text") else "" if txt: parts.append(txt) node = node.next return " ".join(parts).strip() or None return None def _parse_corpus_count(text: str) -> int | None: m = RE_CORPUS_COUNT_WORD.search(text) if not m: return None token = m.group(1).lower() if token.isdigit(): return int(token) return _WORD_NUM.get(token) def _parse_metro(text: str) -> list[JKMetroStation]: stations: list[JKMetroStation] = [] for m in RE_METRO_INLINE.finditer(text): name = m.group(1).strip() # filter common noise if not 2 <= len(name) <= 40 or "ходьб" in name.lower() or "минут" in name.lower(): continue try: stations.append(JKMetroStation(name=name, walk_min=int(m.group(2)))) except ValueError: continue if len(stations) >= 5: break return stations def _extract_jk_address(body_text: str, name: str | None) -> str | None: """Find address line — typically right after JK name, before metro entries.""" if name and name in body_text: after_name = body_text.split(name, 1)[1] # Match 'Город, ул. X / ул. Y' until first metro mention m = re.match( r"\s*([А-ЯЁ][^•]{5,150}?)(?:\s+[А-ЯЁ][а-яё]+\s+\d+\s*мин|\d+\s+оценок|$)", after_name, ) if m: addr = m.group(1).strip().rstrip(",").strip() if 10 < len(addr) < 200: return addr return None # Expose RE_JK_ID at module level for external use (e.g. SERP link → newbuilding detail) __all__ = [ "RE_JK_ID", "JKMetroStation", "YandexNewbuildingInfo", "YandexNewbuildingScraper", ]