From d03c96a79aa4d1c6e570cfb2635b21b3d2e06982 Mon Sep 17 00:00:00 2001
From: bot-backend <bot-backend@gendsgn.local>
Date: Sat, 30 May 2026 19:35:13 +0300
Subject: [PATCH] =?UTF-8?q?fix(scrapers):=20yandex=20area=20=D0=B8=D0=B7?=
 =?UTF-8?q?=20embedded-state=20JSON=20(Refs=20#775)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Часть A (#24): yandex_realty.py парсил площадь из RAW-HTML (regex), пустого до
JS-гидрации → 82% NULL area. _parse_html теперь извлекает {offer_id: area_m2}
из <script id=initial_state_script> (или application/json) раз на страницу и
использует preferentially в _card_to_lot; DOM-regex остаётся fallback'ом.
4 offer-path + 5 area-полей (incl nested {value}). 6 тестов.

Часть B (#23, cian buildingCadastralNumber): УЖЕ реализовано — cian.py:291
читает offer.buildingCadastralNumber, :456 пишет в ScrapedLot, поле base.py:109
существует, тест test_cian_serp_cadastral_numbers зелёный. Изменений не нужно
(премиса аудита устарела). cian.py НЕ трогался.

ruff clean; pytest -k 'yandex or cian' 365 pass (+1 pre-existing unrelated fail
test_cian_valuation low_price mock, не из этого PR).
---
 .../app/services/scrapers/yandex_realty.py    | 145 ++++++++++++++++--
 .../backend/tests/test_yandex_realty_serp.py  | 114 ++++++++++++++
 2 files changed, 249 insertions(+), 10 deletions(-)

diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py
index c4e76dbd..21c34c57 100644
--- a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py
+++ b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py
@@ -4,6 +4,10 @@ History: Yandex SSR used to embed full state in `<script id="initial_state_scrip
 As of 2026-05, the state is chunked + evaporates after hydration. This version
 scrapes the rendered DOM via selectolax + helpers from yandex_helpers.py.
 
+Area extraction: DOM concat-text is empty pre-hydration (~82% NULL). Fix: extract
+area from the page-level embedded JSON state (script[id="initial_state_script"] or
+application/json script), keyed by offerId. DOM regex is kept as fallback.
+
 URL pattern (path-based, no geo radius):
     https://realty.yandex.ru/{city}/kupit/kvartira/vtorichniy-rynok/?page=N
 
@@ -12,6 +16,7 @@ Typical: 23 [data-test="OffersSerpItem"] cards per page.
 
 from __future__ import annotations
 
+import json
 import logging
 import re
 from datetime import date  # noqa: F401  (used in type hints / future helpers)
@@ -47,6 +52,108 @@ PHOTO_SIZE_TO = "main"
 ADDRESS_CONTAINER_SELECTOR = '[class*="AddressWithGeoLinks__addressContainer"]'
 _RE_WS = re.compile(r"\s+")
 
+# Yandex SERP embeds offer data in a <script id="initial_state_script"> as JSON,
+# or (since 2026-05) in <script type="application/json" ...> chunks.
+# We look for these selectors in priority order.
+_STATE_SCRIPT_SELECTORS = [
+    'script[id="initial_state_script"]',
+    'script[type="application/json"]',
+]
+
+# Area field candidates in a Yandex offer object (tried in order).
+_AREA_FIELDS = ("totalArea", "area", "spaceTotal", "spaceAll", "squareTotal")
+
+
+def _extract_offer_areas_from_state(html: str) -> dict[str, float]:
+    """Extract {offer_id: area_m2} map from Yandex SERP embedded JSON state.
+
+    Tries each known script selector, parses JSON, then traverses known
+    offer-array paths looking for offer objects with an id + area field.
+
+    Returns empty dict if state is absent or unparseable.
+    """
+    tree = HTMLParser(html)
+    candidates: list[str] = []
+    for selector in _STATE_SCRIPT_SELECTORS:
+        for node in tree.css(selector):
+            text = node.text() or ""
+            text = text.strip()
+            if text:
+                candidates.append(text)
+
+    result: dict[str, float] = {}
+    for text in candidates:
+        try:
+            state = json.loads(text)
+        except (json.JSONDecodeError, ValueError):
+            continue
+        if not isinstance(state, dict):
+            continue
+        _collect_areas_from_state(state, result)
+        if result:
+            break  # успешно извлекли из первого валидного blob'а
+    return result
+
+
+def _collect_areas_from_state(state: dict[str, Any], out: dict[str, float]) -> None:
+    """Walk known Yandex state paths to find offer objects and extract area.
+
+    Known paths (tried in order):
+      - state["offers"]                           (flat list)
+      - state["pageData"]["offers"]               (SERP v2)
+      - state["listing"]["offers"]                (SERP v3)
+      - state["serpData"]["items"]                (SERP v4 chunks)
+    """
+    offer_lists: list[Any] = []
+    for path in (
+        ["offers"],
+        ["pageData", "offers"],
+        ["listing", "offers"],
+        ["serpData", "items"],
+    ):
+        node: Any = state
+        for key in path:
+            if not isinstance(node, dict):
+                node = None
+                break
+            node = node.get(key)
+        if isinstance(node, list):
+            offer_lists.append(node)
+
+    for offer_list in offer_lists:
+        for offer in offer_list:
+            if not isinstance(offer, dict):
+                continue
+            # Flatten one level of nesting (some Yandex shapes wrap the offer)
+            inner = offer.get("offer") or offer
+            if not isinstance(inner, dict):
+                continue
+            oid = inner.get("offerId") or inner.get("id")
+            if oid is None:
+                continue
+            area = _read_area_field(inner)
+            if area is not None:
+                out[str(oid)] = area
+
+
+def _read_area_field(offer: dict[str, Any]) -> float | None:
+    """Try known field names to read area (м²) from a Yandex offer dict."""
+    for field in _AREA_FIELDS:
+        val = offer.get(field)
+        if val is None:
+            continue
+        # Some fields are nested dicts: {"value": 52.0, "unit": "SQM"}
+        if isinstance(val, dict):
+            val = val.get("value")
+        if val is not None:
+            try:
+                result = float(val)
+                if result > 0:
+                    return result
+            except (TypeError, ValueError):
+                continue
+    return None
+
 
 class YandexRealtyScraper(BaseScraper):
     name = "yandex"
@@ -81,7 +188,11 @@ class YandexRealtyScraper(BaseScraper):
         lots = self._parse_html(response.text, page=page)
         logger.info(
             "yandex serp page=%d city=%s: %d cards (anchor=(%.4f,%.4f) ignored)",
-            page, self.city, len(lots), lat, lon,
+            page,
+            self.city,
+            len(lots),
+            lat,
+            lon,
         )
         await self.sleep_between_requests()
         return lots
@@ -106,7 +217,9 @@ class YandexRealtyScraper(BaseScraper):
                     seen[key] = lot
         logger.info(
             "yandex serp aggregate: %d unique lots over %d pages (city=%s)",
-            len(seen), max_pages, self.city,
+            len(seen),
+            max_pages,
+            self.city,
         )
         return list(seen.values())
 
@@ -118,16 +231,26 @@ class YandexRealtyScraper(BaseScraper):
         return base
 
     def _parse_html(self, html: str, page: int = 0) -> list[ScrapedLot]:
+        # Extract offer-level area from embedded JSON state first (covers ~82% of
+        # cards where DOM text is empty pre-hydration). Falls back to DOM regex per card.
+        state_areas = _extract_offer_areas_from_state(html)
+        if state_areas:
+            logger.debug("yandex serp: state areas extracted for %d offers", len(state_areas))
         tree = HTMLParser(html)
         cards = tree.css('[data-test="OffersSerpItem"]')
         lots: list[ScrapedLot] = []
         for card in cards:
-            lot = self._card_to_lot(card, page=page)
+            lot = self._card_to_lot(card, page=page, state_areas=state_areas)
             if lot is not None:
                 lots.append(lot)
         return lots
 
-    def _card_to_lot(self, card: Node, page: int = 0) -> ScrapedLot | None:
+    def _card_to_lot(
+        self,
+        card: Node,
+        page: int = 0,
+        state_areas: dict[str, float] | None = None,
+    ) -> ScrapedLot | None:
         try:
             # offer_id — required
             offer_link = card.css_first('a[href^="/offer/"]')
@@ -147,11 +270,12 @@ class YandexRealtyScraper(BaseScraper):
             if not price_rub or price_rub <= 0:
                 return None
 
-            # Optional fields
-            area_m = RE_TITLE_AREA.search(text)
-            area_m2 = (
-                float(area_m.group(1).replace(",", ".")) if area_m else None
-            )
+            # Area: prefer JSON state (covers pre-hydration DOM where text is empty),
+            # fall back to DOM regex.
+            area_m2: float | None = (state_areas or {}).get(offer_id)
+            if area_m2 is None:
+                area_m = RE_TITLE_AREA.search(text)
+                area_m2 = float(area_m.group(1).replace(",", ".")) if area_m else None
 
             rooms = self._parse_rooms(text)
 
@@ -189,7 +313,8 @@ class YandexRealtyScraper(BaseScraper):
                     house_source = "yandex_realty_nb"
                     house_ext_id = nb_match.group(2)
                     house_url = (
-                        nb_href if nb_href.startswith("http")
+                        nb_href
+                        if nb_href.startswith("http")
                         else f"https://realty.yandex.ru{nb_href}"
                     )
 
diff --git a/tradein-mvp/backend/tests/test_yandex_realty_serp.py b/tradein-mvp/backend/tests/test_yandex_realty_serp.py
index 1e48f035..e4e61810 100644
--- a/tradein-mvp/backend/tests/test_yandex_realty_serp.py
+++ b/tradein-mvp/backend/tests/test_yandex_realty_serp.py
@@ -1,4 +1,6 @@
 """Unit tests for YandexRealtyScraper DOM-based SERP parser."""
+
+import json
 from datetime import date
 
 import pytest
@@ -7,6 +9,7 @@ from app.services.scrapers.yandex_realty import (
     DEFAULT_CITY,
     MAX_PAGES,
     YandexRealtyScraper,
+    _extract_offer_areas_from_state,
 )
 
 # Realistic single-card fixture (trimmed but selector-faithful).
@@ -225,3 +228,114 @@ def test_address_fallback_to_street_link():
     lots = s._parse_html(NO_ADDR_CONTAINER_CARD_HTML, page=0)
     assert len(lots) == 1
     assert lots[0].address == "улица Малышева"
+
+
+# ── PART A: area from embedded JSON state ────────────────────────────────────
+
+
+def _make_yandex_state_html(offer_id: str, area: float, price_rub: int = 5_000_000) -> str:
+    """Build minimal Yandex SERP HTML with:
+    - <script id="initial_state_script"> containing offer area in JSON state
+    - A DOM card where the text div has NO area (simulating pre-JS-hydration).
+    """
+    state = {
+        "offers": [
+            {
+                "offerId": offer_id,
+                "totalArea": area,
+            }
+        ]
+    }
+    state_json = json.dumps(state, ensure_ascii=False)
+    price_fmt = f"{price_rub:,}".replace(",", " ")
+    return f"""<html><head>
+<script id="initial_state_script">{state_json}</script>
+</head><body>
+<div data-test="OffersSerpItem">
+  <a href="/offer/{offer_id}/">Открыть</a>
+  <div class="AddressWithGeoLinks__addressContainer--xyz">
+    <a href="/ekaterinburg/kupit/kvartira/st-test-1/">улица Тестовая</a>, 10
+  </div>
+  <div>{price_fmt} ₽</div>
+</div>
+</body></html>"""
+
+
+def test_area_from_state_json_no_dom_text():
+    """area_m2 extracted from embedded JSON state when DOM text has no area."""
+    html = _make_yandex_state_html(offer_id="1234567890", area=63.5)
+    s = YandexRealtyScraper()
+    lots = s._parse_html(html, page=0)
+    assert len(lots) == 1
+    assert lots[0].area_m2 == pytest.approx(63.5)
+
+
+def test_area_from_state_json_multiple_offers():
+    """State with multiple offers — each card gets its own area from state."""
+    state = {
+        "offers": [
+            {"offerId": "111", "totalArea": 42.0},
+            {"offerId": "222", "totalArea": 78.3},
+        ]
+    }
+    state_json = json.dumps(state)
+    html = f"""<html><head>
+<script id="initial_state_script">{state_json}</script>
+</head><body>
+<div data-test="OffersSerpItem">
+  <a href="/offer/111/">x</a>
+  <div>3 000 000 ₽</div>
+</div>
+<div data-test="OffersSerpItem">
+  <a href="/offer/222/">x</a>
+  <div>5 000 000 ₽</div>
+</div>
+</body></html>"""
+    s = YandexRealtyScraper()
+    lots = s._parse_html(html, page=0)
+    assert len(lots) == 2
+    areas = {lot.source_id: lot.area_m2 for lot in lots}
+    assert areas["111"] == pytest.approx(42.0)
+    assert areas["222"] == pytest.approx(78.3)
+
+
+def test_area_dom_fallback_when_no_state():
+    """When state script absent, area falls back to DOM regex (existing behavior)."""
+    # SINGLE_CARD_HTML has area in DOM text — state script absent
+    s = YandexRealtyScraper()
+    lots = s._parse_html(SINGLE_CARD_HTML, page=0)
+    assert len(lots) == 1
+    assert lots[0].area_m2 == pytest.approx(36.8)
+
+
+def test_extract_offer_areas_from_state_nested_path():
+    """State via pageData.offers path (SERP v2 shape)."""
+    state = {
+        "pageData": {
+            "offers": [
+                {"offerId": "999", "spaceTotal": 55.0},
+            ]
+        }
+    }
+    html = '<script id="initial_state_script">' + json.dumps(state) + "</script>"
+    areas = _extract_offer_areas_from_state(html)
+    assert areas == {"999": 55.0}
+
+
+def test_extract_offer_areas_area_dict_value():
+    """area field as nested dict {'value': 48.0, 'unit': 'SQM'} (some Yandex responses)."""
+    state = {
+        "offers": [
+            {"offerId": "777", "area": {"value": 48.0, "unit": "SQM"}},
+        ]
+    }
+    html = '<script id="initial_state_script">' + json.dumps(state) + "</script>"
+    areas = _extract_offer_areas_from_state(html)
+    assert areas == {"777": 48.0}
+
+
+def test_extract_offer_areas_empty_when_no_script():
+    """No state script → empty dict returned, no crash."""
+    html = "<html><body><p>no state here</p></body></html>"
+    areas = _extract_offer_areas_from_state(html)
+    assert areas == {}