fix(scrapers): yandex_detail — area/ceiling из структурного блока, покрытие area 63%->~95% (#1792)

tradein-mvp/backend/app/services/scrapers/yandex_detail.py

@@ -58,6 +58,9 @@ class DetailEnrichment(BaseModel):
     title: str | None = None
     rooms: int | None = None
     area_m2: float | None = None
+    living_area_m2: float | None = None
+    kitchen_area_m2: float | None = None
+    ceiling_height: float | None = None  # meters, e.g. 2.55
     floor: int | None = None
     total_floors: int | None = None
 
@@ -166,6 +169,33 @@ class YandexDetailScraper(BaseScraper):
 
         rooms, area_m2, floor, total_floors = _parse_title(title or "")
 
+        # --- Structural offer card (window.INITIAL_STATE → offerCard.card) ---
+        # Authoritative source for area/floor/ceiling/kitchen — the h1 title
+        # misses non-standard layouts (студия / свободная планировка) and never
+        # carries ceiling/kitchen/living. Title stays as fallback below.
+        living_area_m2: float | None = None
+        kitchen_area_m2: float | None = None
+        ceiling_height: float | None = None
+        card = _extract_offer_card(html, offer_id)
+        if card is not None:
+            (
+                c_rooms,
+                c_area,
+                c_living,
+                c_kitchen,
+                c_ceiling,
+                c_floor,
+                c_total_floors,
+            ) = _parse_card_fields(card)
+            # Structural source wins; title only fills the gaps it left.
+            rooms = c_rooms if c_rooms is not None else rooms
+            area_m2 = c_area if c_area is not None else area_m2
+            living_area_m2 = c_living
+            kitchen_area_m2 = c_kitchen
+            ceiling_height = c_ceiling
+            floor = c_floor if c_floor is not None else floor
+            total_floors = c_total_floors if c_total_floors is not None else total_floors
+
         # --- OfferCardSummary text block ---
         summary_node = tree.css_first('[data-test="OfferCardSummary"]')
         summary_text = summary_node.text(strip=True) if summary_node else ""
@@ -243,6 +273,9 @@ class YandexDetailScraper(BaseScraper):
             title=title,
             rooms=rooms,
             area_m2=area_m2,
+            living_area_m2=living_area_m2,
+            kitchen_area_m2=kitchen_area_m2,
+            ceiling_height=ceiling_height,
             floor=floor,
             total_floors=total_floors,
             address=address,
@@ -302,6 +335,148 @@ def _parse_title(title: str) -> tuple[int | None, float | None, int | None, int
     return rooms, area_m2, floor, total_floors
 
 
+def _extract_js_object(html: str, marker: str) -> str | None:
+    """Return the JSON object literal assigned to `marker` (e.g. window.INITIAL_STATE).
+
+    Brace-matches from the first ``{`` after ``marker = ...`` to its balanced
+    close, respecting string literals/escapes. Returns the raw JSON text or None.
+    """
+    i = html.find(marker)
+    if i < 0:
+        return None
+    eq = html.find("=", i)
+    if eq < 0:
+        return None
+    start = html.find("{", eq)
+    if start < 0:
+        return None
+    bal = 0
+    in_str = False
+    esc = False
+    k = start
+    n = len(html)
+    while k < n:
+        c = html[k]
+        if in_str:
+            if esc:
+                esc = False
+            elif c == "\\":
+                esc = True
+            elif c == '"':
+                in_str = False
+        else:
+            if c == '"':
+                in_str = True
+            elif c == "{":
+                bal += 1
+            elif c == "}":
+                bal -= 1
+                if bal == 0:
+                    return html[start : k + 1]
+        k += 1
+    return None
+
+
+def _find_card_by_offer_id(obj: Any, offer_id: str) -> dict[str, Any] | None:
+    """Recursively locate the offer dict whose offerId matches and carries `area`."""
+    if isinstance(obj, dict):
+        if obj.get("offerId") == offer_id and "area" in obj:
+            return obj
+        for v in obj.values():
+            found = _find_card_by_offer_id(v, offer_id)
+            if found is not None:
+                return found
+    elif isinstance(obj, list):
+        for v in obj:
+            found = _find_card_by_offer_id(v, offer_id)
+            if found is not None:
+                return found
+    return None
+
+
+def _extract_offer_card(html: str, offer_id: str) -> dict[str, Any] | None:
+    """Extract the offer card object from window.INITIAL_STATE.
+
+    Yandex embeds full structured offer data in ``window.INITIAL_STATE`` under
+    ``offerCard.card``. This is the authoritative source for area / floor /
+    ceiling / kitchen — unlike the h1 title which misses non-standard layouts
+    and never carries ceiling/kitchen. Returns the card dict or None.
+    """
+    blob = _extract_js_object(html, "window.INITIAL_STATE")
+    if not blob:
+        return None
+    try:
+        state = json.loads(blob)
+    except (json.JSONDecodeError, ValueError):
+        logger.warning("yandex detail: INITIAL_STATE failed to parse for offer %s", offer_id)
+        return None
+    # Fast path: canonical location.
+    offer_card = state.get("offerCard") if isinstance(state, dict) else None
+    if isinstance(offer_card, dict):
+        card = offer_card.get("card")
+        if isinstance(card, dict) and card.get("offerId") == offer_id:
+            return card
+    # Fallback: deep search (page structure may differ).
+    return _find_card_by_offer_id(state, offer_id)
+
+
+def _space_value(node: Any) -> float | None:
+    """Yandex area fields are ``{"value": N, "unit": "SQUARE_METER"}`` → float."""
+    if isinstance(node, dict):
+        val = node.get("value")
+        if isinstance(val, int | float):
+            return float(val)
+    elif isinstance(node, int | float):
+        return float(node)
+    return None
+
+
+def _parse_card_fields(
+    card: dict[str, Any],
+) -> tuple[
+    int | None,
+    float | None,
+    float | None,
+    float | None,
+    float | None,
+    int | None,
+    int | None,
+]:
+    """Extract (rooms, area, living, kitchen, ceiling, floor, total_floors) from card.
+
+    rooms: ``roomsTotal``, or 0 when ``house.studio`` is truthy (студии не
+    несут roomsTotal). floor: first element of ``floorsOffered``.
+    """
+    area = _space_value(card.get("area"))
+    living = _space_value(card.get("livingSpace"))
+    kitchen = _space_value(card.get("kitchenSpace"))
+
+    ceiling_raw = card.get("ceilingHeight")
+    ceiling: float | None = None
+    if isinstance(ceiling_raw, int | float):
+        ceiling = float(ceiling_raw)
+
+    rooms_raw = card.get("roomsTotal")
+    rooms: int | None = None
+    if isinstance(rooms_raw, int):
+        rooms = rooms_raw
+    house = card.get("house")
+    if isinstance(house, dict) and house.get("studio"):
+        rooms = 0
+
+    total_floors_raw = card.get("floorsTotal")
+    total_floors = total_floors_raw if isinstance(total_floors_raw, int) else None
+
+    floor: int | None = None
+    floors_offered = card.get("floorsOffered")
+    if isinstance(floors_offered, list) and floors_offered:
+        first = floors_offered[0]
+        if isinstance(first, int):
+            floor = first
+
+    return rooms, area, living, kitchen, ceiling, floor, total_floors
+
+
 def _find_section_text(tree: HTMLParser, heading: str) -> str | None:
     """Find the text content of a <section>/<div> whose preceding h2/h3 matches heading.
 
@@ -400,6 +575,18 @@ def save_detail_enrichment(db: Session, listing_id: int, e: DetailEnrichment) ->
             UPDATE listings SET
                 rooms                = COALESCE(CAST(:rooms AS int),        rooms),
                 area_m2              = COALESCE(CAST(:area_m2 AS numeric),  area_m2),
+                living_area_m2       = COALESCE(
+                                           CAST(:living_area_m2 AS numeric),
+                                           living_area_m2
+                                       ),
+                kitchen_area_m2      = COALESCE(
+                                           CAST(:kitchen_area_m2 AS numeric),
+                                           kitchen_area_m2
+                                       ),
+                ceiling_height       = COALESCE(
+                                           CAST(:ceiling_height AS numeric),
+                                           ceiling_height
+                                       ),
                 floor                = COALESCE(CAST(:floor AS int),        floor),
                 total_floors         = COALESCE(CAST(:total_floors AS int), total_floors),
                 address              = COALESCE(CAST(:address AS text),     address),
@@ -443,6 +630,9 @@ def save_detail_enrichment(db: Session, listing_id: int, e: DetailEnrichment) ->
             "listing_id": listing_id,
             "rooms": e.rooms,
             "area_m2": e.area_m2,
+            "living_area_m2": e.living_area_m2,
+            "kitchen_area_m2": e.kitchen_area_m2,
+            "ceiling_height": e.ceiling_height,
             "floor": e.floor,
             "total_floors": e.total_floors,
             "address": e.address,

tradein-mvp/backend/tests/fixtures/yandex_offer_3402418396407468801.html

@@ -0,0 +1,7 @@
+<!DOCTYPE html><html><head>
+<script>window.INITIAL_STATE = {"offerCard": {"card": {"kitchenSpace": {"value": 9.4, "unit": "SQUARE_METER"}, "floorsTotal": 9, "floorsOffered": [4], "roomsTotal": 1, "offerId": "3402418396407468801", "price": {"value": 3850000}, "house": {"studio": false}, "area": {"value": 35.5, "unit": "SQUARE_METER"}, "description": "Пpодаётся уютная однокомнатная квapтиpa 35.5 кв.м на 4 этаже 9-ти этажного кирпичного дома.\nПлaнировка удoбнaя и функционaльная. \nПpосторная жилая комнатa 16.9 кв.м. правильной формы пoзвoляeт эффективно распределить жилое пространство. Kухня 9.4 кв.м изолирoвaнная и вместительная - разместит необходимую бытовую технику. Отдельная кладовая/гapдeробная 1.4 кв.м помогает грамотно организовать хранение вещей и сохранить порядок в квартире. Аккуратный и совмещенный санузел. Окна выходят на солнечную сторону.\nВ квартире установлены пластиковые окна (в т.ч. на балконе), счётчики на воду, домофон, пр", "flatType": "SECONDARY", "ceilingHeight": 2.55, "livingSpace": {"value": 16.9, "unit": "SQUARE_METER"}}}};</script>
+<script type="application/ld+json">{"@context": "https://schema.org", "@type": "Product", "category": "APARTMENT", "description": "Продается однокомнатная квартира от собственника 35.5 м² по цене 3,85 млн ₽ на 4 этаже из 9. 📍 Адрес: Екатеринбург, улица Титова, 25А, ✅ Характеристики, описание и фото объекта id 3402418396407468801 на Яндекс Недвижимости", "image": ["//avatars.mds.yandex.net/get-realty-offers/17720277/add.0b8cfb034a87f5daa92c312e95adedcf.realty-api-vos/app_large", "//avatars.mds.yandex.net/get-realty-offers/17720277/add.0b8cfb034a87f5daa92c312e95adedcf.realty-api-vos/large", "//avatars.mds.yandex.net/get-realty-offers/17720277/add.0b8cfb034a87f5daa92c312e95adedcf.realty-api-vos/app_snippet_large", "//avatars.mds.yandex.net/get-realty-offers/17720277/add.0b8cfb034a87f5daa92c312e95adedcf.realty-api-vos/app_middle", "//avatars.mds.yandex.net/get-realty-offers/17720277/add.0b8cfb034a87f5daa92c312e95adedcf.realty-api-vos/app_snippet_middle", "//avatars.mds.yandex.net/get-realty-offers/17720277/add.0b8cfb034a87f5daa92c312e95adedcf.realty-api-vos/app_snippet_mini", "//avatars.mds.yandex.net/get-realty-offers/17720277/add.0b8cfb034a87f5daa92c312e95adedcf.realty-api-vos/app_snippet_small", "//avatars.mds.yandex.net/get-realty-offers/17720277/add.0b8cfb034a87f5daa92c312e95adedcf.realty-api-vos/cosmic"], "name": "Купить 1-комнатную квартиру от собственника 35.5 м², 4 этаж из 9, за 3,85 млн ₽ — Екатеринбург, улица Титова, 25А — id 3402418396407468801", "offers": {"@type": "Offer", "availability": "https://schema.org/InStock", "price": 3850000, "priceCurrency": "RUR", "url": "https://realty.yandex.ru/offer/3402418396407468801"}, "sku": "3402418396407468801"}</script>
+</head><body>
+<h1>35,5 м², 1-комнатная квартира</h1>
+<div data-test="OfferCardSummary">16 июня, 8 просмотров35,5 м², 1-комнатная квартира3 850 000 ₽108 451 ₽ за м²альтернативаПоказать телефон +7 (×××) ×××-××-××НаписатьЕкатеринбург, улица Титова, 25АЧкаловская14 мин.ещё 2 станцииАнжелаРазмещено собственникомДобавить заметку</div>
+</body></html>

tradein-mvp/backend/tests/fixtures/yandex_offer_7811691822126445498.html

@@ -0,0 +1,7 @@
+<!DOCTYPE html><html><head>
+<script>window.INITIAL_STATE = {"offerCard": {"card": {"area": {"value": 24.92, "unit": "SQUARE_METER"}, "price": {"value": 4838168}, "floorsTotal": 25, "floorsOffered": [9], "offerId": "7811691822126445498", "flatType": "NEW_FLAT", "house": {"studio": true}, "description": "IV очередь «Тёплых кварталов»: квартира с ремонтом, рассрочка до сдачи дома, специальная ипотека от Гринвич недвижимость. Сдача в III квартале 2026 года\nЧистовая отделка «под ключ»\n•\tЛаминат 32 класса, обои под покраску, потолки с отделкой.\n•\tВ санузлах уже установлены умывальник со смесителем и унитаз.\n•\tСчётчики на электричество, воду и тепло\nИпотека от застройщика «Гринвич недвижимость» — ставка ниже рынка. Узнайте подробнее в отделе продаж\nРассрочка без ежемесячных платежей до конца строительства. Фирменная рассрочка Гринвич недвижимость.\n«Тёплые кварталы» — это не просто дом, а целый квар"}}};</script>
+<script type="application/ld+json">{"@context": "https://schema.org", "@type": "Product", "category": "APARTMENT", "description": "Продается квартира-студию 24.92 м² по цене 4,84 млн ₽ на 9 этаже из 25. 📍 Адрес: Екатеринбург, жилой район Втузгородок, ЖК Тёплые Кварталы, ЖК «Тёплые кварталы». ✅ Характеристики, описание и фото объекта id 7811691822126445498 на Яндекс Недвижимости", "image": ["//avatars.mds.yandex.net/get-realty-offers/11390665/62da87c5-c166-499b-9cd1-e79323378bb2/app_large", "//avatars.mds.yandex.net/get-realty-offers/11390665/62da87c5-c166-499b-9cd1-e79323378bb2/large", "//avatars.mds.yandex.net/get-realty-offers/11390665/62da87c5-c166-499b-9cd1-e79323378bb2/app_snippet_large", "//avatars.mds.yandex.net/get-realty-offers/11390665/62da87c5-c166-499b-9cd1-e79323378bb2/app_middle", "//avatars.mds.yandex.net/get-realty-offers/11390665/62da87c5-c166-499b-9cd1-e79323378bb2/app_snippet_middle", "//avatars.mds.yandex.net/get-realty-offers/11390665/62da87c5-c166-499b-9cd1-e79323378bb2/app_snippet_mini", "//avatars.mds.yandex.net/get-realty-offers/11390665/62da87c5-c166-499b-9cd1-e79323378bb2/app_snippet_small", "//avatars.mds.yandex.net/get-realty-offers/11390665/62da87c5-c166-499b-9cd1-e79323378bb2/cosmic"], "name": "Купить квартиру-студию 24.92 м², 9 этаж из 25, за 4,84 млн ₽ — ЖК «Тёплые кварталы», Екатеринбург, жилой район Втузгородок, ЖК Тёплые Кварталы — id 7811691822126445498", "offers": {"@type": "Offer", "availability": "https://schema.org/InStock", "price": 4838168, "priceCurrency": "RUR", "url": "https://realty.yandex.ru/offer/7811691822126445498"}, "sku": "7811691822126445498"}</script>
+</head><body>
+<h1>24,9 м², квартира-студия</h1>
+<div data-test="OfferCardSummary">вчера24,9 м², квартира-студия4 838 168 ₽194 148 ₽ за м²срок сдачи 3 кв. 2026 г.первичная продажаПоказать телефон +7 (×××) ×××-××-××Екатеринбург, жилой район Втузгородок, ЖК Тёплые КварталыУральская22 мин.ещё 3 станцииЗастройщик «Паритет Девелопмент»3 сдано8 строятсяДобавить заметку</div>
+</body></html>

tradein-mvp/backend/tests/test_yandex_detail_structural.py

@@ -0,0 +1,153 @@
+"""Tests for YandexDetailScraper structural-source extraction (#1792).
+
+Area / ceiling / kitchen / living / floor are extracted from the embedded
+``window.INITIAL_STATE → offerCard.card`` object rather than the h1 title.
+The title misses non-standard layouts (студия / свободная планировка) and
+never carries ceiling/kitchen — which is why DB area coverage sat at ~63%.
+
+Fixtures are trimmed real offer pages fetched via the headless browser:
+  - yandex_offer_3402418396407468801.html — 1-комн. вторичка (full chars)
+  - yandex_offer_7811691822126445498.html — студия / новостройка
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from app.services.scrapers.yandex_detail import (
+    YandexDetailScraper,
+    _extract_offer_card,
+    _parse_card_fields,
+)
+
+FIXTURES = Path(__file__).parent / "fixtures"
+SCRAPER = YandexDetailScraper()
+
+# offer_id -> expected structural values
+_SECONDARY_ID = "3402418396407468801"
+_STUDIO_ID = "7811691822126445498"
+
+
+def _load(offer_id: str) -> str:
+    return (FIXTURES / f"yandex_offer_{offer_id}.html").read_text(encoding="utf-8")
+
+
+def _url(offer_id: str) -> str:
+    return f"https://realty.yandex.ru/offer/{offer_id}/"
+
+
+# ---------------------------------------------------------------------------
+# Secondary (1-комнатная) — все структурные поля присутствуют
+# ---------------------------------------------------------------------------
+
+
+class TestSecondaryOfferStructural:
+    def test_area_from_structural_source(self) -> None:
+        r = SCRAPER.parse(_load(_SECONDARY_ID), offer_url=_url(_SECONDARY_ID))
+        assert r is not None
+        assert r.area_m2 == pytest.approx(35.5)
+
+    def test_rooms_floor_from_structural_source(self) -> None:
+        r = SCRAPER.parse(_load(_SECONDARY_ID), offer_url=_url(_SECONDARY_ID))
+        assert r is not None
+        assert r.rooms == 1
+        # floor from floorsOffered[0]; total from floorsTotal
+        assert r.floor == 4
+        assert r.total_floors == 9
+
+    def test_ceiling_kitchen_living_present(self) -> None:
+        r = SCRAPER.parse(_load(_SECONDARY_ID), offer_url=_url(_SECONDARY_ID))
+        assert r is not None
+        # These never come from the title — only the structural block carries them.
+        assert r.ceiling_height == pytest.approx(2.55)
+        assert r.kitchen_area_m2 == pytest.approx(9.4)
+        assert r.living_area_m2 == pytest.approx(16.9)
+
+
+# ---------------------------------------------------------------------------
+# Studio / новостройка — area present even when title says «квартира-студия»
+# ---------------------------------------------------------------------------
+
+
+class TestStudioOfferStructural:
+    def test_area_extracted_for_studio(self) -> None:
+        r = SCRAPER.parse(_load(_STUDIO_ID), offer_url=_url(_STUDIO_ID))
+        assert r is not None
+        # The key regression: studio area must NOT be None.
+        assert r.area_m2 is not None
+        assert r.area_m2 == pytest.approx(24.92)
+
+    def test_studio_rooms_zero_from_house_flag(self) -> None:
+        r = SCRAPER.parse(_load(_STUDIO_ID), offer_url=_url(_STUDIO_ID))
+        assert r is not None
+        # roomsTotal is null for studios; house.studio == true → rooms = 0.
+        assert r.rooms == 0
+
+    def test_studio_floor_extracted(self) -> None:
+        r = SCRAPER.parse(_load(_STUDIO_ID), offer_url=_url(_STUDIO_ID))
+        assert r is not None
+        assert r.floor == 9
+        assert r.total_floors == 25
+
+    def test_studio_ceiling_kitchen_legitimately_absent(self) -> None:
+        # This is a newbuilding offer — Yandex does not expose ceiling/kitchen/
+        # living for it. None here is legitimate, not a parse miss.
+        r = SCRAPER.parse(_load(_STUDIO_ID), offer_url=_url(_STUDIO_ID))
+        assert r is not None
+        assert r.ceiling_height is None
+        assert r.kitchen_area_m2 is None
+        assert r.living_area_m2 is None
+
+
+# ---------------------------------------------------------------------------
+# Helper-level coverage
+# ---------------------------------------------------------------------------
+
+
+class TestStructuralHelpers:
+    def test_extract_offer_card_matches_offer_id(self) -> None:
+        card = _extract_offer_card(_load(_SECONDARY_ID), _SECONDARY_ID)
+        assert card is not None
+        assert card["offerId"] == _SECONDARY_ID
+
+    def test_extract_offer_card_missing_state_returns_none(self) -> None:
+        assert _extract_offer_card("<html><body>no state</body></html>", "123") is None
+
+    def test_parse_card_fields_space_objects(self) -> None:
+        card = {
+            "roomsTotal": 2,
+            "area": {"value": 55.3, "unit": "SQUARE_METER"},
+            "livingSpace": {"value": 30.0, "unit": "SQUARE_METER"},
+            "kitchenSpace": {"value": 12.5, "unit": "SQUARE_METER"},
+            "ceilingHeight": 2.7,
+            "floorsTotal": 18,
+            "floorsOffered": [7],
+            "house": {"studio": False},
+        }
+        rooms, area, living, kitchen, ceiling, floor, total = _parse_card_fields(card)
+        assert rooms == 2
+        assert area == pytest.approx(55.3)
+        assert living == pytest.approx(30.0)
+        assert kitchen == pytest.approx(12.5)
+        assert ceiling == pytest.approx(2.7)
+        assert floor == 7
+        assert total == 18
+
+    def test_parse_card_fields_studio_flag_sets_rooms_zero(self) -> None:
+        card = {
+            "roomsTotal": None,
+            "area": {"value": 24.92, "unit": "SQUARE_METER"},
+            "floorsOffered": [9],
+            "floorsTotal": 25,
+            "house": {"studio": True},
+        }
+        rooms, area, living, kitchen, ceiling, floor, total = _parse_card_fields(card)
+        assert rooms == 0
+        assert area == pytest.approx(24.92)
+        assert living is None
+        assert kitchen is None
+        assert ceiling is None
+        assert floor == 9
+        assert total == 25