Merge pull request 'feat(tradein): yandex_realty.py — DOM SERP refactor (vtorichka, 23 cards/page)' (#462) from feat/tradein-yandex-serp-refactor into main

tradein-mvp/backend/app/services/scrapers/yandex_realty.py

@@ -1,290 +1,234 @@
-"""Yandex.Недвижимость scraper (realty.yandex.ru).
+"""Yandex.Недвижимость scraper (realty.yandex.ru) — DOM-based SERP parser.
 
-Я.Недвижимость отдаёт SSR HTML с `__INITIAL_STATE__` JSON в `<script>` теге.
-Структура offers: state.search.offers — list of offers.
+History: Yandex SSR used to embed full state in `<script id="initial_state_script">`.
+As of 2026-05, the state is chunked + evaporates after hydration. This version
+scrapes the rendered DOM via selectolax + helpers from yandex_helpers.py.
 
-URL pattern:
-  https://realty.yandex.ru/ekaterinburg/kupit/kvartira/?
-    rgid=240970               # ЕКБ region
-    &offerType=SELL
-    &category=APARTMENT
-    &geoLat=56.83
-    &geoLon=60.59
-    &geoRadius=1000           # метры
+URL pattern (path-based, no geo radius):
+    https://realty.yandex.ru/{city}/kupit/kvartira/vtorichniy-rynok/?page=N
+
+Typical: 23 [data-test="OffersSerpItem"] cards per page.
 """
 
 from __future__ import annotations
 
-import json
 import logging
-import random
-import re
+from datetime import date  # noqa: F401  (used in type hints / future helpers)
 from typing import Any
-from urllib.parse import urlencode, urljoin
 
-from selectolax.parser import HTMLParser
+from selectolax.parser import HTMLParser, Node
 
 from app.services.scrapers.base import BaseScraper, ScrapedLot
+from app.services.scrapers.yandex_helpers import (
+    RE_FLOOR,
+    RE_JK_ID,
+    RE_OFFER_ID,
+    RE_PPM2,
+    RE_PRICE,
+    RE_TITLE_AREA,
+    RE_TITLE_ROOMS,
+    parse_ru_date,
+    parse_rub,
+)
 
 logger = logging.getLogger(__name__)
 
-YANDEX_EKB_RGID = 240970
+DEFAULT_CITY = "ekaterinburg"
+MAX_PAGES = 3
+PHOTO_DOMAIN = "avatars.mds.yandex"
+PHOTO_SIZE_FROM = "app_snippet_small"
+PHOTO_SIZE_TO = "main"
 
 
 class YandexRealtyScraper(BaseScraper):
-    """Yandex.Недвижимость parser."""
-
     name = "yandex"
     base_url = "https://realty.yandex.ru"
     request_delay_sec = 5.0
 
+    def __init__(self, city: str = DEFAULT_CITY) -> None:
+        super().__init__()
+        self.city = city
+
     async def fetch_around(
         self,
         lat: float,
         lon: float,
         radius_m: int = 1000,
-        rooms: str | None = None,
-        sort: str = "DATE_DESC",
         page: int = 0,
     ) -> list[ScrapedLot]:
-        self._anchor_lat = lat
-        self._anchor_lon = lon
-        self._anchor_radius_m = radius_m
-
-        url = self._build_url(lat, lon, radius_m, rooms, sort, page)
+        """Fetch ONE page of SERP cards. lat/lon/radius_m ignored (Yandex uses
+        path-based vtorichka URL); kept for BaseScraper compat + logging.
+        """
+        url = self._build_url(page=page)
         try:
             response = await self._http_get(url)
         except Exception:
-            logger.exception("yandex realty fetch failed for %s", url)
+            logger.exception("yandex serp fetch failed: %s", url)
             return []
-
         if response.status_code != 200:
-            logger.warning("yandex realty returned %d", response.status_code)
+            logger.warning("yandex serp returned %d for %s", response.status_code, url)
             return []
-
-        lots = self._parse_html(response.text)
+        lots = self._parse_html(response.text, page=page)
         logger.info(
-            "yandex realty: %d lots around (%.4f, %.4f) rooms=%s sort=%s page=%d",
-            len(lots), lat, lon, rooms or "ANY", sort, page,
+            "yandex serp page=%d city=%s: %d cards (anchor=(%.4f,%.4f) ignored)",
+            page, self.city, len(lots), lat, lon,
         )
         await self.sleep_between_requests()
         return lots
 
     async def fetch_around_multi_room(
-        self, lat: float, lon: float, radius_m: int = 1000,
-        sorts: tuple[str, ...] = ("DATE_DESC",),
-        pages: tuple[int, ...] = (0,),
-    ) -> list[ScrapedLot]:
-        """Скрейп Yandex по сегментам STUDIO/1k/2k/3k/4k+ и опционально по
-        нескольким sort/page вариантам.
-
-        Каждая выдача = 20 лотов. Полный обход rooms×sorts×pages даст
-        5 × len(sorts) × len(pages) × 20 потенциальных лотов до дедупликации.
-        """
-        seen: dict[str, ScrapedLot] = {}
-        for rooms in ("STUDIO", "1", "2", "3", "PLUS_4"):
-            for sort in sorts:
-                for page in pages:
-                    try:
-                        lots = await self.fetch_around(
-                            lat, lon, radius_m, rooms=rooms, sort=sort, page=page,
-                        )
-                    except Exception:
-                        logger.exception(
-                            "yandex realty multi-room fetch failed: rooms=%s sort=%s page=%d",
-                            rooms, sort, page,
-                        )
-                        continue
-                    for lot in lots:
-                        key = lot.source_id or lot.source_url
-                        if key and key not in seen:
-                            seen[key] = lot
-        logger.info(
-            "yandex realty multi-room: %d unique lots around (%.4f, %.4f)",
-            len(seen), lat, lon,
-        )
-        return list(seen.values())
-
-    def _build_url(
         self,
         lat: float,
         lon: float,
-        radius_m: int,
-        rooms: str | None = None,
-        sort: str = "DATE_DESC",
-        page: int = 0,
-    ) -> str:
-        params: dict[str, Any] = {
-            "rgid": YANDEX_EKB_RGID,
-            "offerType": "SELL",
-            "category": "APARTMENT",
-            "geoLat": f"{lat:.4f}",
-            "geoLon": f"{lon:.4f}",
-            "geoRadius": radius_m,
-            "sort": sort,
-        }
-        if rooms:
-            # Yandex Realty принимает roomsTotal=STUDIO|1|2|3|PLUS_4
-            params["roomsTotal"] = rooms
+        radius_m: int = 1000,
+        max_pages: int = MAX_PAGES,
+        **_legacy_kwargs: Any,  # swallow rooms/sorts/pages from old callers
+    ) -> list[ScrapedLot]:
+        """Fetch up to max_pages of SERP cards, dedup by offer_id (source_id)."""
+        seen: dict[str, ScrapedLot] = {}
+        for page in range(max_pages):
+            lots = await self.fetch_around(lat, lon, radius_m, page=page)
+            if not lots:
+                break  # empty page → no more results
+            for lot in lots:
+                key = lot.source_id or lot.source_url
+                if key and key not in seen:
+                    seen[key] = lot
+        logger.info(
+            "yandex serp aggregate: %d unique lots over %d pages (city=%s)",
+            len(seen), max_pages, self.city,
+        )
+        return list(seen.values())
+
+    def _build_url(self, page: int = 0) -> str:
+        # Yandex paginates 1-based; `page=0` → no param (first page)
+        base = f"{self.base_url}/{self.city}/kupit/kvartira/vtorichniy-rynok/"
         if page > 0:
-            params["page"] = page
-        return f"{self.base_url}/ekaterinburg/kupit/kvartira/?{urlencode(params)}"
+            return f"{base}?page={page}"
+        return base
 
-    def _parse_html(self, html: str) -> list[ScrapedLot]:
-        # Поиск __INITIAL_STATE__ или __PRELOADED_STATE__
-        state = _extract_yandex_state(html)
-        if not state:
-            return []
-        offers = _items_from_state(state)
-        return [
-            lot for lot in (self._item_to_lot(o) for o in offers if isinstance(o, dict))
-            if lot is not None
-        ]
+    def _parse_html(self, html: str, page: int = 0) -> list[ScrapedLot]:
+        tree = HTMLParser(html)
+        cards = tree.css('[data-test="OffersSerpItem"]')
+        lots: list[ScrapedLot] = []
+        for card in cards:
+            lot = self._card_to_lot(card, page=page)
+            if lot is not None:
+                lots.append(lot)
+        return lots
 
-    def _item_to_lot(self, offer: dict[str, Any]) -> ScrapedLot | None:
+    def _card_to_lot(self, card: Node, page: int = 0) -> ScrapedLot | None:
         try:
-            # Цена: offer.price.value (там целое число рублей)
-            price_obj = offer.get("price") or {}
-            price = price_obj.get("value")
-            if not price:
-                pf = price_obj.get("priceForWhole") or {}
-                price = pf.get("value") if isinstance(pf, dict) else pf
-            if not price:
+            # offer_id — required
+            offer_link = card.css_first('a[href^="/offer/"]')
+            if offer_link is None:
+                return None
+            href = offer_link.attributes.get("href", "")
+            id_match = RE_OFFER_ID.search(href)
+            if not id_match:
+                return None
+            offer_id = id_match.group(1)
+
+            text = card.text(strip=True)
+
+            # Price — required (ScrapedLot.price_rub > 0)
+            price_m = RE_PRICE.search(text)
+            price_rub = parse_rub(price_m.group(1)) if price_m else None
+            if not price_rub or price_rub <= 0:
                 return None
 
-            # Площадь
-            area_obj = offer.get("area") or {}
-            area = area_obj.get("value")
-            if area:
-                area = float(area)
-
-            rooms = offer.get("roomsTotal") or offer.get("rooms")
-
-            # Этажи — offer.floorsOffered = [N], building.floorsTotal
-            floors_offered = offer.get("floorsOffered") or []
-            floor = floors_offered[0] if floors_offered else offer.get("floor")
-            building = offer.get("building") or {}
-            total_floors = building.get("floorsTotal") or offer.get("floorsTotal")
-            year_built = building.get("builtYear")
-            wall_type = building.get("buildingType")
-
-            # Локация
-            loc = offer.get("location") or {}
-            address = (
-                loc.get("address")
-                or loc.get("geocoderAddress")
-                or "Екатеринбург (Я.Недвижимость)"
+            # Optional fields
+            area_m = RE_TITLE_AREA.search(text)
+            area_m2 = (
+                float(area_m.group(1).replace(",", ".")) if area_m else None
             )
-            point = loc.get("point") or {}
-            lat = point.get("latitude")
-            lon = point.get("longitude")
 
-            # Anchor fallback (если EXACT нет — но обычно есть)
-            if (lat is None or lon is None) and hasattr(self, "_anchor_lat"):
-                jitter_deg = (self._anchor_radius_m / 1000) / 111
-                lat = self._anchor_lat + (random.random() - 0.5) * jitter_deg
-                lon = self._anchor_lon + (random.random() - 0.5) * jitter_deg / 2
+            rooms = self._parse_rooms(text)
 
-            # URL
-            offer_id = offer.get("offerId") or offer.get("id")
-            internal_url = offer.get("unsignedInternalUrl") or ""
-            if internal_url.startswith("//"):
-                full_url = "https:" + internal_url
-            elif offer_id:
-                full_url = f"https://realty.yandex.ru/offer/{offer_id}/"
-            else:
-                return None
+            floor_m = RE_FLOOR.search(text)
+            floor = int(floor_m.group(1)) if floor_m else None
+            total_floors = int(floor_m.group(2)) if floor_m else None
+
+            ppm2_m = RE_PPM2.search(text)
+            price_per_m2 = parse_rub(ppm2_m.group(1)) if ppm2_m else None
+
+            bargain = "торг" in text.lower()
+            listing_date = parse_ru_date(text)
+
+            # Address — street aggregator link
+            street_link = card.css_first('a[href*="/kupit/kvartira/st-"]')
+            address = street_link.text(strip=True) if street_link else None
 
             # Photos
-            photos = []
-            for img_field in ("appLargeImages", "extImages", "alikeImages"):
-                for img in (offer.get(img_field) or [])[:5]:
-                    if isinstance(img, dict):
-                        url = img.get("xl") or img.get("large") or img.get("main", "")
-                        if url:
-                            photos.append(url)
-                    elif isinstance(img, str):
-                        photos.append(img)
-                if photos:
-                    break
+            photo_urls = self._extract_photos(card)
+
+            # Newbuilding linkage
+            house_source: str | None = None
+            house_ext_id: str | None = None
+            house_url: str | None = None
+            nb_link = card.css_first('a[href*="/kupit/novostrojka/"]')
+            if nb_link is not None:
+                nb_href = nb_link.attributes.get("href", "")
+                nb_match = RE_JK_ID.search(nb_href)
+                if nb_match:
+                    house_source = "yandex_realty_nb"
+                    house_ext_id = nb_match.group(2)
+                    house_url = (
+                        nb_href if nb_href.startswith("http")
+                        else f"https://realty.yandex.ru{nb_href}"
+                    )
 
             return ScrapedLot(
-                source="yandex",
-                source_url=full_url,
-                source_id=str(offer_id) if offer_id else None,
+                source=self.name,
+                source_url=f"https://realty.yandex.ru/offer/{offer_id}/",
+                source_id=offer_id,
                 address=address,
-                lat=lat,
-                lon=lon,
+                lat=None,
+                lon=None,
                 rooms=rooms,
-                area_m2=area,
+                area_m2=area_m2,
                 floor=floor,
                 total_floors=total_floors,
-                year_built=year_built,
-                house_type=_map_wall_type(wall_type),
-                price_rub=int(price),
-                photo_urls=photos[:5],
+                price_rub=price_rub,
+                price_per_m2=price_per_m2,
+                bargain_allowed=bargain,
+                listing_date=listing_date,
+                photo_urls=photo_urls,
+                house_source=house_source,
+                house_ext_id=house_ext_id,
+                house_url=house_url,
+                listing_segment="vtorichka",
                 raw_payload={
-                    # Compact dump (full offer 50KB+, тяжеловато для БД)
-                    "offerId": offer_id,
-                    "creationDate": offer.get("creationDate"),
-                    "dealStatus": offer.get("dealStatus"),
-                    "description": (offer.get("description") or "")[:500],
+                    "card_text": text[:1000],
+                    "page": page,
+                    "city": self.city,
                 },
             )
         except Exception:
-            logger.exception("yandex item parse failed: %s", offer.get("offerId"))
+            logger.exception("yandex card parse failed (page=%d)", page)
             return None
 
-
-# ── Helpers ──────────────────────────────────────────────────────────────────
-
-
-_RE_STATE = re.compile(
-    r'<script id="initial_state_script"[^>]*>(.+?)</script>',
-    re.DOTALL,
-)
-
-
-def _extract_yandex_state(html: str) -> dict[str, Any] | None:
-    m = _RE_STATE.search(html)
-    if not m:
-        return None
-    raw = m.group(1)
-    # Может начинаться с `window.__INITIAL_STATE__=` префикса
-    raw = re.sub(r'^\s*window\.[\w_]+\s*=\s*', '', raw).rstrip(';').strip()
-    try:
-        return json.loads(raw)
-    except json.JSONDecodeError:
+    @staticmethod
+    def _parse_rooms(text: str) -> int | None:
+        m = RE_TITLE_ROOMS.search(text)
+        if not m:
+            return None
+        if m.group(2):  # studio
+            return 0
+        if m.group(1):  # numbered
+            try:
+                return int(m.group(1))
+            except ValueError:
+                return None
         return None
 
-
-def _items_from_state(state: dict[str, Any]) -> list[dict[str, Any]]:
-    """Yandex Realty: state.map.offers.points содержит список объявлений (десктоп desktop SSR)."""
-    for path in [
-        ("map", "offers", "points"),    # верифицировано на проде
-        ("search", "offers"),
-        ("offers", "points"),
-        ("offers", "items"),
-    ]:
-        cursor: Any = state
-        for key in path:
-            if not isinstance(cursor, dict):
-                cursor = None
+    @staticmethod
+    def _extract_photos(card: Node) -> list[str]:
+        urls: list[str] = []
+        for img in card.css("img"):
+            src = img.attributes.get("src", "") or ""
+            if PHOTO_DOMAIN in src:
+                urls.append(src.replace(PHOTO_SIZE_FROM, PHOTO_SIZE_TO))
+            if len(urls) >= 5:
                 break
-            cursor = cursor.get(key)
-        if isinstance(cursor, list) and cursor:
-            return cursor
-    return []
-
-
-def _map_wall_type(yandex_type: str | None) -> str | None:
-    if not yandex_type:
-        return None
-    mapping = {
-        "PANEL": "panel",
-        "BRICK": "brick",
-        "MONOLITH": "monolith",
-        "MONOLITH_BRICK": "monolith_brick",
-        "BLOCK": "panel",
-    }
-    return mapping.get(yandex_type.upper(), "other")
+        return urls

tradein-mvp/backend/tests/test_yandex_realty_serp.py

@@ -0,0 +1,136 @@
+"""Unit tests for YandexRealtyScraper DOM-based SERP parser."""
+from datetime import date
+
+import pytest
+
+from app.services.scrapers.yandex_realty import (
+    DEFAULT_CITY,
+    MAX_PAGES,
+    YandexRealtyScraper,
+)
+
+# Realistic single-card fixture (trimmed but selector-faithful).
+# Note: selectolax.text(strip=True) concatenates all text nodes without
+# added separators — so floor/price data lives in a single line div with
+# middot (·) separators, matching real Yandex SERP card layout.
+SINGLE_CARD_HTML = """
+<html><body>
+<div data-test="OffersSerpItem">
+  <a href="/offer/7567094292504417257/">Открыть</a>
+  <a href="/ekaterinburg/kupit/kvartira/st-frezerovshchikov-12345/">Фрезеровщиков</a>
+  <a href="/ekaterinburg/kupit/novostrojka/tatlin-1592987/">ЖК Татлин</a>
+  <img src="https://avatars.mds.yandex.net/get-realty/12345/app_snippet_small/test.jpg">
+  <img src="https://avatars.mds.yandex.net/get-realty/67890/app_snippet_small/test2.jpg">
+  <img src="https://other-domain.example/photo.jpg">
+  <div>36,8 м² · 1-комнатная квартира · 8 этаж из 9 · 4 399 000 ₽ · 119 566 ₽ за м²</div>
+  <div>торг возможен · опубликовано 17 февраля 2026</div>
+</div>
+</body></html>
+"""
+
+STUDIO_CARD_HTML = """
+<div data-test="OffersSerpItem">
+  <a href="/offer/9999999/">x</a>
+  <div>Студия · 28 м² · 2 этаж из 9 · 3 200 000 ₽</div>
+</div>
+"""
+
+NO_PRICE_CARD_HTML = """
+<div data-test="OffersSerpItem">
+  <a href="/offer/1111/">x</a>
+  <div>40 м² · 1-комнатная · 3 этаж из 5</div>
+</div>
+"""
+
+EMPTY_PAGE_HTML = "<html><body><div>Нет объявлений</div></body></html>"
+
+
+def test_max_pages_default():
+    assert MAX_PAGES == 3
+
+
+def test_default_city():
+    assert DEFAULT_CITY == "ekaterinburg"
+
+
+def test_build_url_first_page():
+    s = YandexRealtyScraper()
+    assert s._build_url(page=0) == (
+        "https://realty.yandex.ru/ekaterinburg/kupit/kvartira/vtorichniy-rynok/"
+    )
+
+
+def test_build_url_paginated():
+    s = YandexRealtyScraper()
+    assert s._build_url(page=2) == (
+        "https://realty.yandex.ru/ekaterinburg/kupit/kvartira/vtorichniy-rynok/?page=2"
+    )
+
+
+def test_build_url_other_city():
+    s = YandexRealtyScraper(city="moscow")
+    assert s._build_url(page=0) == (
+        "https://realty.yandex.ru/moscow/kupit/kvartira/vtorichniy-rynok/"
+    )
+
+
+def test_parse_html_extracts_card():
+    s = YandexRealtyScraper()
+    lots = s._parse_html(SINGLE_CARD_HTML, page=0)
+    assert len(lots) == 1
+    lot = lots[0]
+    assert lot.source == "yandex"
+    assert lot.source_id == "7567094292504417257"
+    assert lot.source_url == "https://realty.yandex.ru/offer/7567094292504417257/"
+    assert lot.address == "Фрезеровщиков"
+    assert lot.area_m2 == pytest.approx(36.8)
+    assert lot.rooms == 1
+    assert lot.floor == 8
+    assert lot.total_floors == 9
+    assert lot.price_rub == 4_399_000
+    assert lot.price_per_m2 == 119_566
+    assert lot.bargain_allowed is True
+    assert lot.listing_date == date(2026, 2, 17)
+    assert lot.house_source == "yandex_realty_nb"
+    assert lot.house_ext_id == "1592987"
+    assert lot.house_url == (
+        "https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/"
+    )
+    assert lot.listing_segment == "vtorichka"
+    assert lot.lat is None and lot.lon is None  # SERP has no coords
+    # Photos: 2 from yandex CDN, the third (other-domain) skipped
+    assert len(lot.photo_urls) == 2
+    assert all("avatars.mds.yandex" in u for u in lot.photo_urls)
+    # Size upgrade applied: app_snippet_small → main
+    assert all("app_snippet_small" not in u for u in lot.photo_urls)
+    assert all("main" in u for u in lot.photo_urls)
+
+
+def test_parse_html_studio_rooms_zero():
+    s = YandexRealtyScraper()
+    lots = s._parse_html(STUDIO_CARD_HTML, page=0)
+    assert len(lots) == 1
+    assert lots[0].rooms == 0  # studio convention
+    assert lots[0].area_m2 == pytest.approx(28.0)
+    assert lots[0].floor == 2 and lots[0].total_floors == 9
+
+
+def test_parse_html_no_price_skipped():
+    s = YandexRealtyScraper()
+    lots = s._parse_html(NO_PRICE_CARD_HTML, page=0)
+    assert lots == []
+
+
+def test_parse_html_empty_page():
+    s = YandexRealtyScraper()
+    lots = s._parse_html(EMPTY_PAGE_HTML, page=0)
+    assert lots == []
+
+
+def test_parse_html_multi_card():
+    multi = SINGLE_CARD_HTML + STUDIO_CARD_HTML
+    s = YandexRealtyScraper()
+    lots = s._parse_html(multi, page=1)
+    assert len(lots) == 2
+    assert {lot.source_id for lot in lots} == {"7567094292504417257", "9999999"}
+    assert all(lot.raw_payload["page"] == 1 for lot in lots)