"""Yandex Realty detail page scraper. Fetches /offer// and extracts Product JSON-LD + DOM sections into a DetailEnrichment Pydantic model. Used by enrichment pipeline (Wave 5+ matching / Wave 6 estimator). """ from __future__ import annotations import json import logging import re from datetime import date from typing import Any from pydantic import BaseModel, Field from selectolax.parser import HTMLParser, Node from sqlalchemy import text from sqlalchemy.orm import Session from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper from app.services.scrapers.repair_state_normalizer import infer_repair_state_from_text from app.services.scrapers.yandex_helpers import ( RE_AGENCY_FOUNDED, RE_AGENCY_OBJECTS, RE_METRO_WALK, RE_VIEWS, RE_YEAR, find_ld_by_type, parse_house_type, parse_ru_date, parse_rub, ) logger = logging.getLogger(__name__) # ── Pydantic models ─────────────────────────────────────────────────────────── class MetroStation(BaseModel): name: str walk_min: int | None = None class DetailEnrichment(BaseModel): """Enrichment payload from a Yandex detail page.""" offer_id: str source_url: str # Pricing — Product JSON-LD `offers.price` is the exact int price_rub: int | None = None price_per_m2: int | None = None # Title + basic params title: str | None = None rooms: int | None = None area_m2: float | None = None living_area_m2: float | None = None kitchen_area_m2: float | None = None ceiling_height: float | None = None # meters, e.g. 2.55 floor: int | None = None total_floors: int | None = None # Address (full) address: str | None = None # Description (full text) description: str | None = None # Repair state — enum, inferred from description text (#622). # Yandex не отдаёт структурного поля ремонта, поэтому только инференс. repair_state: str | None = None # Stats views_total: int | None = None publish_date: date | None = None publish_date_relative: str | None = None # Agency block (OfferCardAuthorInfo) agency_name: str | None = None agency_founded_year: int | None = None agency_objects_count: int | None = None seller_name: str | None = None # last text line before "Агентство «...»" # Metro stations from "Расположение" section metro_stations: list[MetroStation] = Field(default_factory=list) # Photos — 8 sizes from Product.image[] photo_urls: list[str] = Field(default_factory=list) # Newbuilding linkage newbuilding_url: str | None = None newbuilding_id: str | None = None # NLP from description (best-effort) house_type_nlp: str | None = None year_built_hint: int | None = None metro_walk_min: int | None = None # Raw payload (trimmed) raw_payload: dict[str, Any] | None = None # ── Scraper ─────────────────────────────────────────────────────────────────── class YandexDetailScraper(BaseScraper): """Detail page scraper for realty.yandex.ru.""" name = "yandex_detail" base_url = "https://realty.yandex.ru" request_delay_sec = 5.0 # class default; instance value loaded from scraper_settings def __init__(self) -> None: super().__init__() self.request_delay_sec = get_scraper_delay(self.name) # BaseScraper requires fetch_around — detail isn't geo-based, raise NotImplementedError async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override] raise NotImplementedError( "YandexDetailScraper is offer-id-based; use fetch_detail(offer_url) instead." ) async def fetch_detail(self, offer_url: str) -> DetailEnrichment | None: try: response = await self._http_get(offer_url) except Exception: logger.exception("yandex detail fetch failed: %s", offer_url) return None if response.status_code != 200: logger.warning("yandex detail returned %d for %s", response.status_code, offer_url) return None result = self.parse(response.text, offer_url=offer_url) await self.sleep_between_requests() return result def parse(self, html: str, offer_url: str) -> DetailEnrichment | None: offer_id_match = re.search(r"/offer/(\d+)/?", offer_url) if not offer_id_match: logger.warning("offer_url has no /offer//: %s", offer_url) return None offer_id = offer_id_match.group(1) tree = HTMLParser(html) # --- Product JSON-LD (authoritative price) --- product = find_ld_by_type(html, "Product") or {} offers_ld = product.get("offers") or {} if isinstance(offers_ld, list) and offers_ld: offers_ld = offers_ld[0] price_ld = offers_ld.get("price") if isinstance(offers_ld, dict) else None try: price_rub = int(price_ld) if price_ld else None except (TypeError, ValueError): price_rub = None # Photos from JSON-LD image[] (typically 8 size variants) images = product.get("image") or [] if isinstance(images, str): images = [images] photo_urls = [u for u in images if isinstance(u, str)] # --- Title + summary --- title_node = tree.css_first("h1") title = title_node.text(strip=True) if title_node else None rooms, area_m2, floor, total_floors = _parse_title(title or "") # --- Structural offer card (window.INITIAL_STATE → offerCard.card) --- # Authoritative source for area/floor/ceiling/kitchen — the h1 title # misses non-standard layouts (студия / свободная планировка) and never # carries ceiling/kitchen/living. Title stays as fallback below. living_area_m2: float | None = None kitchen_area_m2: float | None = None ceiling_height: float | None = None card = _extract_offer_card(html, offer_id) if card is not None: ( c_rooms, c_area, c_living, c_kitchen, c_ceiling, c_floor, c_total_floors, ) = _parse_card_fields(card) # Structural source wins; title only fills the gaps it left. rooms = c_rooms if c_rooms is not None else rooms area_m2 = c_area if c_area is not None else area_m2 living_area_m2 = c_living kitchen_area_m2 = c_kitchen ceiling_height = c_ceiling floor = c_floor if c_floor is not None else floor total_floors = c_total_floors if c_total_floors is not None else total_floors # --- OfferCardSummary text block --- summary_node = tree.css_first('[data-test="OfferCardSummary"]') summary_text = summary_node.text(strip=True) if summary_node else "" # Views + relative publish date from summary text views_match = RE_VIEWS.search(summary_text) views_total = int(views_match.group(1)) if views_match else None publish_date = parse_ru_date(summary_text) publish_date_relative = _extract_relative_date(summary_text) # price_per_m2 — from summary text if absent in LD price_per_m2: int | None = None ppm2_match = re.search(r"(\d[\d\s]+)\s*₽\s*за\s*м²", summary_text) if ppm2_match: price_per_m2 = parse_rub(ppm2_match.group(1)) # --- OfferCardAuthorInfo (agency block) --- author_node = tree.css_first('[data-test="OfferCardAuthorInfo"]') agency_name: str | None = None agency_founded_year: int | None = None agency_objects_count: int | None = None seller_name: str | None = None if author_node is not None: author_text = author_node.text(strip=True) agency_h2 = author_node.css_first("h2") agency_name = agency_h2.text(strip=True) if agency_h2 else None founded_m = RE_AGENCY_FOUNDED.search(author_text) if founded_m: agency_founded_year = int(founded_m.group(1)) objects_m = RE_AGENCY_OBJECTS.search(author_text) if objects_m: agency_objects_count = int(objects_m.group(1)) # seller_name — last text line before "Агентство" seller_name = _extract_seller_name(summary_text, agency_name) # --- Description section (after H2 "Описание") --- description = _find_section_text(tree, "Описание") # --- Repair state: инференс из описания (#622), Yandex без структурного поля --- repair_state = infer_repair_state_from_text(description or summary_text) # --- Address --- address = _extract_address(summary_text) # --- Metro stations from "Расположение" section --- location_text = _find_section_text(tree, "Расположение") or "" metro_stations = _parse_metro_stations(location_text) # --- Newbuilding link --- nb_url: str | None = None nb_id: str | None = None nb_link = tree.css_first('a[href*="/kupit/novostrojka/"]') if nb_link is not None: nb_href = nb_link.attributes.get("href", "") nb_match = re.search(r"/novostrojka/[\w-]+?-(\d+)/?", nb_href) if nb_match: nb_id = nb_match.group(1) nb_url = ( nb_href if nb_href.startswith("http") else f"https://realty.yandex.ru{nb_href}" ) # --- NLP best-effort from description --- nlp_text = description or summary_text house_type_nlp = parse_house_type(nlp_text) year_hint_m = RE_YEAR.search(nlp_text or "") year_built_hint = int(year_hint_m.group(1)) if year_hint_m else None walk_m = RE_METRO_WALK.search(nlp_text or "") metro_walk_min = int(walk_m.group(1)) if walk_m else None return DetailEnrichment( offer_id=offer_id, source_url=offer_url, price_rub=price_rub, price_per_m2=price_per_m2, title=title, rooms=rooms, area_m2=area_m2, living_area_m2=living_area_m2, kitchen_area_m2=kitchen_area_m2, ceiling_height=ceiling_height, floor=floor, total_floors=total_floors, address=address, description=description, repair_state=repair_state, views_total=views_total, publish_date=publish_date, publish_date_relative=publish_date_relative, agency_name=agency_name, agency_founded_year=agency_founded_year, agency_objects_count=agency_objects_count, seller_name=seller_name, metro_stations=metro_stations, photo_urls=photo_urls, newbuilding_url=nb_url, newbuilding_id=nb_id, house_type_nlp=house_type_nlp, year_built_hint=year_built_hint, metro_walk_min=metro_walk_min, raw_payload={ "summary_text": summary_text[:1000], "description_len": len(description) if description else 0, "photo_count": len(photo_urls), }, ) # ── Helpers ─────────────────────────────────────────────────────────────────── def _parse_title(title: str) -> tuple[int | None, float | None, int | None, int | None]: """Extract (rooms, area_m2, floor, total_floors) from h1 text.""" rooms: int | None = None area_m2: float | None = None floor: int | None = None total_floors: int | None = None area_m = re.search(r"(\d+[.,]?\d*)\s*м²", title) if area_m: area_m2 = float(area_m.group(1).replace(",", ".")) if re.search(r"студи[яюй]", title, re.IGNORECASE): rooms = 0 else: rooms_m = re.search(r"(\d+)\s*-?\s*комнатн", title, re.IGNORECASE) if rooms_m: try: rooms = int(rooms_m.group(1)) except ValueError: pass floor_m = re.search(r"(\d+)\s+этаж\s+из\s+(\d+)", title, re.IGNORECASE) if floor_m: floor = int(floor_m.group(1)) total_floors = int(floor_m.group(2)) return rooms, area_m2, floor, total_floors def _extract_js_object(html: str, marker: str) -> str | None: """Return the JSON object literal assigned to `marker` (e.g. window.INITIAL_STATE). Brace-matches from the first ``{`` after ``marker = ...`` to its balanced close, respecting string literals/escapes. Returns the raw JSON text or None. """ i = html.find(marker) if i < 0: return None eq = html.find("=", i) if eq < 0: return None start = html.find("{", eq) if start < 0: return None bal = 0 in_str = False esc = False k = start n = len(html) while k < n: c = html[k] if in_str: if esc: esc = False elif c == "\\": esc = True elif c == '"': in_str = False else: if c == '"': in_str = True elif c == "{": bal += 1 elif c == "}": bal -= 1 if bal == 0: return html[start : k + 1] k += 1 return None def _find_card_by_offer_id(obj: Any, offer_id: str) -> dict[str, Any] | None: """Recursively locate the offer dict whose offerId matches and carries `area`.""" if isinstance(obj, dict): if obj.get("offerId") == offer_id and "area" in obj: return obj for v in obj.values(): found = _find_card_by_offer_id(v, offer_id) if found is not None: return found elif isinstance(obj, list): for v in obj: found = _find_card_by_offer_id(v, offer_id) if found is not None: return found return None def _extract_offer_card(html: str, offer_id: str) -> dict[str, Any] | None: """Extract the offer card object from window.INITIAL_STATE. Yandex embeds full structured offer data in ``window.INITIAL_STATE`` under ``offerCard.card``. This is the authoritative source for area / floor / ceiling / kitchen — unlike the h1 title which misses non-standard layouts and never carries ceiling/kitchen. Returns the card dict or None. """ blob = _extract_js_object(html, "window.INITIAL_STATE") if not blob: return None try: state = json.loads(blob) except (json.JSONDecodeError, ValueError): logger.warning("yandex detail: INITIAL_STATE failed to parse for offer %s", offer_id) return None # Fast path: canonical location. offer_card = state.get("offerCard") if isinstance(state, dict) else None if isinstance(offer_card, dict): card = offer_card.get("card") if isinstance(card, dict) and card.get("offerId") == offer_id: return card # Fallback: deep search (page structure may differ). return _find_card_by_offer_id(state, offer_id) def _space_value(node: Any) -> float | None: """Yandex area fields are ``{"value": N, "unit": "SQUARE_METER"}`` → float.""" if isinstance(node, dict): val = node.get("value") if isinstance(val, int | float): return float(val) elif isinstance(node, int | float): return float(node) return None def _parse_card_fields( card: dict[str, Any], ) -> tuple[ int | None, float | None, float | None, float | None, float | None, int | None, int | None, ]: """Extract (rooms, area, living, kitchen, ceiling, floor, total_floors) from card. rooms: ``roomsTotal``, or 0 when ``house.studio`` is truthy (студии не несут roomsTotal). floor: first element of ``floorsOffered``. """ area = _space_value(card.get("area")) living = _space_value(card.get("livingSpace")) kitchen = _space_value(card.get("kitchenSpace")) ceiling_raw = card.get("ceilingHeight") ceiling: float | None = None if isinstance(ceiling_raw, int | float): ceiling = float(ceiling_raw) rooms_raw = card.get("roomsTotal") rooms: int | None = None if isinstance(rooms_raw, int): rooms = rooms_raw house = card.get("house") if isinstance(house, dict) and house.get("studio"): rooms = 0 total_floors_raw = card.get("floorsTotal") total_floors = total_floors_raw if isinstance(total_floors_raw, int) else None floor: int | None = None floors_offered = card.get("floorsOffered") if isinstance(floors_offered, list) and floors_offered: first = floors_offered[0] if isinstance(first, int): floor = first return rooms, area, living, kitchen, ceiling, floor, total_floors def _find_section_text(tree: HTMLParser, heading: str) -> str | None: """Find the text content of a
/
whose preceding h2/h3 matches heading. Yandex page structure varies; this scans h2/h3 nodes, then returns the concatenated text of subsequent sibling blocks until the next heading. """ for h in tree.css("h2, h3"): if heading.lower() in (h.text(strip=True) or "").lower(): # collect subsequent siblings until the next h2/h3 parts: list[str] = [] node: Node | None = h.next while node is not None: tag = (node.tag or "").lower() if tag in {"h2", "h3"}: break txt = node.text(strip=True) if hasattr(node, "text") else "" if txt: parts.append(txt) node = node.next return " ".join(parts).strip() or None return None def _extract_address(summary_text: str) -> str | None: """Best-effort address extraction from summary block.""" # Pattern: "Россия, Свердловская область, Екатеринбург, улица Х, д. N" m = re.search(r"(Россия[^•]+?)(?:•|\d+\s+просмотр|$)", summary_text) if m: addr = m.group(1).strip().rstrip(",").strip() return addr if len(addr) > 10 else None return None def _parse_metro_stations(location_text: str) -> list[MetroStation]: """Parse 'Уральская 11 мин. Динамо 16 мин.' → list of MetroStation.""" stations: list[MetroStation] = [] # name (1+ Cyrillic words) + space + N + space + мин(.|у|ут) for m in re.finditer(r"([А-ЯЁ][А-Яа-яё\s-]+?)\s+(\d+)\s*мин", location_text): name = m.group(1).strip() if 2 <= len(name) <= 40: stations.append(MetroStation(name=name, walk_min=int(m.group(2)))) if len(stations) >= 5: break return stations def _extract_relative_date(summary_text: str) -> str | None: """Capture phrases like '6 часов назад' / 'вчера' / '3 дня назад'.""" m = re.search( r"(\d+\s+(?:минут|час|часов|часа|день|дня|дней|недел[ьюи])\s+назад" r"|вчера|сегодня|позавчера)", summary_text, re.IGNORECASE, ) return m.group(1).strip() if m else None def _extract_seller_name(summary_text: str, agency_name: str | None) -> str | None: """Heuristic: line right before 'Агентство ...' in summary text.""" if not agency_name or agency_name not in summary_text: return None head = summary_text.split(agency_name, 1)[0] # last short token sequence (likely "Имя Фамилия") m = re.findall(r"([А-ЯЁ][а-яё]+(?:\s+[А-ЯЁ][а-яё]+){1,2})", head) return m[-1] if m else None # ── Save helper ─────────────────────────────────────────────────────────────── def save_detail_enrichment(db: Session, listing_id: int, e: DetailEnrichment) -> bool: """Persist Yandex DetailEnrichment to listings row. UPDATE listings SET = COALESCE(:val, ), ..., detail_enriched_at = NOW() WHERE id = listing_id. COALESCE semantics: keeps existing non-NULL value if new value is NULL (never overwrites a populated column with NULL). area_m2 from detail is more accurate than SERP, but COALESCE preserves SERP value if detail returns NULL — acceptable. Returns True if the UPDATE touched at least one row (listing_id found in DB). """ metro_json: str | None = None if e.metro_stations: metro_json = json.dumps( [s.model_dump(exclude_none=True) for s in e.metro_stations], ensure_ascii=False, ) photo_json: str | None = None if e.photo_urls: photo_json = json.dumps(e.photo_urls, ensure_ascii=False) result = db.execute( text(""" UPDATE listings SET rooms = COALESCE(CAST(:rooms AS int), rooms), area_m2 = COALESCE(CAST(:area_m2 AS numeric), area_m2), living_area_m2 = COALESCE( CAST(:living_area_m2 AS numeric), living_area_m2 ), kitchen_area_m2 = COALESCE( CAST(:kitchen_area_m2 AS numeric), kitchen_area_m2 ), ceiling_height = COALESCE( CAST(:ceiling_height AS numeric), ceiling_height ), floor = COALESCE(CAST(:floor AS int), floor), total_floors = COALESCE(CAST(:total_floors AS int), total_floors), address = COALESCE(CAST(:address AS text), address), description = COALESCE(CAST(:description AS text), description), repair_state = COALESCE(CAST(:repair_state AS text),repair_state), publish_date = COALESCE(CAST(:publish_date AS date),publish_date), views_total_yandex = COALESCE(CAST(:views_total AS int), views_total_yandex), publish_date_relative= COALESCE( CAST(:pub_date_rel AS text), publish_date_relative ), agency_name = COALESCE(CAST(:agency_name AS text), agency_name), agency_founded_year = COALESCE( CAST(:agency_founded_year AS int), agency_founded_year ), agency_objects_count = COALESCE( CAST(:agency_objects_count AS int), agency_objects_count ), metro_stations = COALESCE( CAST(:metro_stations AS jsonb), metro_stations ), photo_urls = COALESCE( CAST(:photo_urls AS jsonb), photo_urls ), newbuilding_url = COALESCE( CAST(:newbuilding_url AS text), newbuilding_url ), newbuilding_id = COALESCE( CAST(:newbuilding_id AS text), newbuilding_id ), detail_enriched_at = NOW() WHERE id = CAST(:listing_id AS bigint) """), { "listing_id": listing_id, "rooms": e.rooms, "area_m2": e.area_m2, "living_area_m2": e.living_area_m2, "kitchen_area_m2": e.kitchen_area_m2, "ceiling_height": e.ceiling_height, "floor": e.floor, "total_floors": e.total_floors, "address": e.address, "description": e.description, "repair_state": e.repair_state, "publish_date": e.publish_date, "views_total": e.views_total, "pub_date_rel": e.publish_date_relative, "agency_name": e.agency_name, "agency_founded_year": e.agency_founded_year, "agency_objects_count": e.agency_objects_count, "metro_stations": metro_json, "photo_urls": photo_json, "newbuilding_url": e.newbuilding_url, "newbuilding_id": e.newbuilding_id, }, ) db.commit() saved = (result.rowcount or 0) > 0 logger.info( "yandex detail enrichment saved listing_id=%s (metro=%d photos=%d saved=%s)", listing_id, len(e.metro_stations), len(e.photo_urls), saved, ) return saved