diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_detail.py b/tradein-mvp/backend/app/services/scrapers/yandex_detail.py index 2850ee30..08b88a45 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_detail.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_detail.py @@ -58,6 +58,9 @@ class DetailEnrichment(BaseModel): title: str | None = None rooms: int | None = None area_m2: float | None = None + living_area_m2: float | None = None + kitchen_area_m2: float | None = None + ceiling_height: float | None = None # meters, e.g. 2.55 floor: int | None = None total_floors: int | None = None @@ -166,6 +169,33 @@ class YandexDetailScraper(BaseScraper): rooms, area_m2, floor, total_floors = _parse_title(title or "") + # --- Structural offer card (window.INITIAL_STATE → offerCard.card) --- + # Authoritative source for area/floor/ceiling/kitchen — the h1 title + # misses non-standard layouts (студия / свободная планировка) and never + # carries ceiling/kitchen/living. Title stays as fallback below. + living_area_m2: float | None = None + kitchen_area_m2: float | None = None + ceiling_height: float | None = None + card = _extract_offer_card(html, offer_id) + if card is not None: + ( + c_rooms, + c_area, + c_living, + c_kitchen, + c_ceiling, + c_floor, + c_total_floors, + ) = _parse_card_fields(card) + # Structural source wins; title only fills the gaps it left. + rooms = c_rooms if c_rooms is not None else rooms + area_m2 = c_area if c_area is not None else area_m2 + living_area_m2 = c_living + kitchen_area_m2 = c_kitchen + ceiling_height = c_ceiling + floor = c_floor if c_floor is not None else floor + total_floors = c_total_floors if c_total_floors is not None else total_floors + # --- OfferCardSummary text block --- summary_node = tree.css_first('[data-test="OfferCardSummary"]') summary_text = summary_node.text(strip=True) if summary_node else "" @@ -243,6 +273,9 @@ class YandexDetailScraper(BaseScraper): title=title, rooms=rooms, area_m2=area_m2, + living_area_m2=living_area_m2, + kitchen_area_m2=kitchen_area_m2, + ceiling_height=ceiling_height, floor=floor, total_floors=total_floors, address=address, @@ -302,6 +335,148 @@ def _parse_title(title: str) -> tuple[int | None, float | None, int | None, int return rooms, area_m2, floor, total_floors +def _extract_js_object(html: str, marker: str) -> str | None: + """Return the JSON object literal assigned to `marker` (e.g. window.INITIAL_STATE). + + Brace-matches from the first ``{`` after ``marker = ...`` to its balanced + close, respecting string literals/escapes. Returns the raw JSON text or None. + """ + i = html.find(marker) + if i < 0: + return None + eq = html.find("=", i) + if eq < 0: + return None + start = html.find("{", eq) + if start < 0: + return None + bal = 0 + in_str = False + esc = False + k = start + n = len(html) + while k < n: + c = html[k] + if in_str: + if esc: + esc = False + elif c == "\\": + esc = True + elif c == '"': + in_str = False + else: + if c == '"': + in_str = True + elif c == "{": + bal += 1 + elif c == "}": + bal -= 1 + if bal == 0: + return html[start : k + 1] + k += 1 + return None + + +def _find_card_by_offer_id(obj: Any, offer_id: str) -> dict[str, Any] | None: + """Recursively locate the offer dict whose offerId matches and carries `area`.""" + if isinstance(obj, dict): + if obj.get("offerId") == offer_id and "area" in obj: + return obj + for v in obj.values(): + found = _find_card_by_offer_id(v, offer_id) + if found is not None: + return found + elif isinstance(obj, list): + for v in obj: + found = _find_card_by_offer_id(v, offer_id) + if found is not None: + return found + return None + + +def _extract_offer_card(html: str, offer_id: str) -> dict[str, Any] | None: + """Extract the offer card object from window.INITIAL_STATE. + + Yandex embeds full structured offer data in ``window.INITIAL_STATE`` under + ``offerCard.card``. This is the authoritative source for area / floor / + ceiling / kitchen — unlike the h1 title which misses non-standard layouts + and never carries ceiling/kitchen. Returns the card dict or None. + """ + blob = _extract_js_object(html, "window.INITIAL_STATE") + if not blob: + return None + try: + state = json.loads(blob) + except (json.JSONDecodeError, ValueError): + logger.warning("yandex detail: INITIAL_STATE failed to parse for offer %s", offer_id) + return None + # Fast path: canonical location. + offer_card = state.get("offerCard") if isinstance(state, dict) else None + if isinstance(offer_card, dict): + card = offer_card.get("card") + if isinstance(card, dict) and card.get("offerId") == offer_id: + return card + # Fallback: deep search (page structure may differ). + return _find_card_by_offer_id(state, offer_id) + + +def _space_value(node: Any) -> float | None: + """Yandex area fields are ``{"value": N, "unit": "SQUARE_METER"}`` → float.""" + if isinstance(node, dict): + val = node.get("value") + if isinstance(val, int | float): + return float(val) + elif isinstance(node, int | float): + return float(node) + return None + + +def _parse_card_fields( + card: dict[str, Any], +) -> tuple[ + int | None, + float | None, + float | None, + float | None, + float | None, + int | None, + int | None, +]: + """Extract (rooms, area, living, kitchen, ceiling, floor, total_floors) from card. + + rooms: ``roomsTotal``, or 0 when ``house.studio`` is truthy (студии не + несут roomsTotal). floor: first element of ``floorsOffered``. + """ + area = _space_value(card.get("area")) + living = _space_value(card.get("livingSpace")) + kitchen = _space_value(card.get("kitchenSpace")) + + ceiling_raw = card.get("ceilingHeight") + ceiling: float | None = None + if isinstance(ceiling_raw, int | float): + ceiling = float(ceiling_raw) + + rooms_raw = card.get("roomsTotal") + rooms: int | None = None + if isinstance(rooms_raw, int): + rooms = rooms_raw + house = card.get("house") + if isinstance(house, dict) and house.get("studio"): + rooms = 0 + + total_floors_raw = card.get("floorsTotal") + total_floors = total_floors_raw if isinstance(total_floors_raw, int) else None + + floor: int | None = None + floors_offered = card.get("floorsOffered") + if isinstance(floors_offered, list) and floors_offered: + first = floors_offered[0] + if isinstance(first, int): + floor = first + + return rooms, area, living, kitchen, ceiling, floor, total_floors + + def _find_section_text(tree: HTMLParser, heading: str) -> str | None: """Find the text content of a
/
whose preceding h2/h3 matches heading. @@ -400,6 +575,18 @@ def save_detail_enrichment(db: Session, listing_id: int, e: DetailEnrichment) -> UPDATE listings SET rooms = COALESCE(CAST(:rooms AS int), rooms), area_m2 = COALESCE(CAST(:area_m2 AS numeric), area_m2), + living_area_m2 = COALESCE( + CAST(:living_area_m2 AS numeric), + living_area_m2 + ), + kitchen_area_m2 = COALESCE( + CAST(:kitchen_area_m2 AS numeric), + kitchen_area_m2 + ), + ceiling_height = COALESCE( + CAST(:ceiling_height AS numeric), + ceiling_height + ), floor = COALESCE(CAST(:floor AS int), floor), total_floors = COALESCE(CAST(:total_floors AS int), total_floors), address = COALESCE(CAST(:address AS text), address), @@ -443,6 +630,9 @@ def save_detail_enrichment(db: Session, listing_id: int, e: DetailEnrichment) -> "listing_id": listing_id, "rooms": e.rooms, "area_m2": e.area_m2, + "living_area_m2": e.living_area_m2, + "kitchen_area_m2": e.kitchen_area_m2, + "ceiling_height": e.ceiling_height, "floor": e.floor, "total_floors": e.total_floors, "address": e.address, diff --git a/tradein-mvp/backend/tests/fixtures/yandex_offer_3402418396407468801.html b/tradein-mvp/backend/tests/fixtures/yandex_offer_3402418396407468801.html new file mode 100644 index 00000000..748d4bbf --- /dev/null +++ b/tradein-mvp/backend/tests/fixtures/yandex_offer_3402418396407468801.html @@ -0,0 +1,7 @@ + + + + +

35,5 м², 1-комнатная квартира

+
16 июня, 8 просмотров35,5 м², 1-комнатная квартира3 850 000 ₽108 451 ₽ за м²альтернативаПоказать телефон +7 (×××) ×××-××-××НаписатьЕкатеринбург, улица Титова, 25АЧкаловская14 мин.ещё 2 станцииАнжелаРазмещено собственникомДобавить заметку
+ diff --git a/tradein-mvp/backend/tests/fixtures/yandex_offer_7811691822126445498.html b/tradein-mvp/backend/tests/fixtures/yandex_offer_7811691822126445498.html new file mode 100644 index 00000000..dd6f9c58 --- /dev/null +++ b/tradein-mvp/backend/tests/fixtures/yandex_offer_7811691822126445498.html @@ -0,0 +1,7 @@ + + + + +

24,9 м², квартира-студия

+
вчера24,9 м², квартира-студия4 838 168 ₽194 148 ₽ за м²срок сдачи 3 кв. 2026 г.первичная продажаПоказать телефон +7 (×××) ×××-××-××Екатеринбург, жилой район Втузгородок, ЖК Тёплые КварталыУральская22 мин.ещё 3 станцииЗастройщик «Паритет Девелопмент»3 сдано8 строятсяДобавить заметку
+ diff --git a/tradein-mvp/backend/tests/test_yandex_detail_structural.py b/tradein-mvp/backend/tests/test_yandex_detail_structural.py new file mode 100644 index 00000000..4547f519 --- /dev/null +++ b/tradein-mvp/backend/tests/test_yandex_detail_structural.py @@ -0,0 +1,153 @@ +"""Tests for YandexDetailScraper structural-source extraction (#1792). + +Area / ceiling / kitchen / living / floor are extracted from the embedded +``window.INITIAL_STATE → offerCard.card`` object rather than the h1 title. +The title misses non-standard layouts (студия / свободная планировка) and +never carries ceiling/kitchen — which is why DB area coverage sat at ~63%. + +Fixtures are trimmed real offer pages fetched via the headless browser: + - yandex_offer_3402418396407468801.html — 1-комн. вторичка (full chars) + - yandex_offer_7811691822126445498.html — студия / новостройка +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from app.services.scrapers.yandex_detail import ( + YandexDetailScraper, + _extract_offer_card, + _parse_card_fields, +) + +FIXTURES = Path(__file__).parent / "fixtures" +SCRAPER = YandexDetailScraper() + +# offer_id -> expected structural values +_SECONDARY_ID = "3402418396407468801" +_STUDIO_ID = "7811691822126445498" + + +def _load(offer_id: str) -> str: + return (FIXTURES / f"yandex_offer_{offer_id}.html").read_text(encoding="utf-8") + + +def _url(offer_id: str) -> str: + return f"https://realty.yandex.ru/offer/{offer_id}/" + + +# --------------------------------------------------------------------------- +# Secondary (1-комнатная) — все структурные поля присутствуют +# --------------------------------------------------------------------------- + + +class TestSecondaryOfferStructural: + def test_area_from_structural_source(self) -> None: + r = SCRAPER.parse(_load(_SECONDARY_ID), offer_url=_url(_SECONDARY_ID)) + assert r is not None + assert r.area_m2 == pytest.approx(35.5) + + def test_rooms_floor_from_structural_source(self) -> None: + r = SCRAPER.parse(_load(_SECONDARY_ID), offer_url=_url(_SECONDARY_ID)) + assert r is not None + assert r.rooms == 1 + # floor from floorsOffered[0]; total from floorsTotal + assert r.floor == 4 + assert r.total_floors == 9 + + def test_ceiling_kitchen_living_present(self) -> None: + r = SCRAPER.parse(_load(_SECONDARY_ID), offer_url=_url(_SECONDARY_ID)) + assert r is not None + # These never come from the title — only the structural block carries them. + assert r.ceiling_height == pytest.approx(2.55) + assert r.kitchen_area_m2 == pytest.approx(9.4) + assert r.living_area_m2 == pytest.approx(16.9) + + +# --------------------------------------------------------------------------- +# Studio / новостройка — area present even when title says «квартира-студия» +# --------------------------------------------------------------------------- + + +class TestStudioOfferStructural: + def test_area_extracted_for_studio(self) -> None: + r = SCRAPER.parse(_load(_STUDIO_ID), offer_url=_url(_STUDIO_ID)) + assert r is not None + # The key regression: studio area must NOT be None. + assert r.area_m2 is not None + assert r.area_m2 == pytest.approx(24.92) + + def test_studio_rooms_zero_from_house_flag(self) -> None: + r = SCRAPER.parse(_load(_STUDIO_ID), offer_url=_url(_STUDIO_ID)) + assert r is not None + # roomsTotal is null for studios; house.studio == true → rooms = 0. + assert r.rooms == 0 + + def test_studio_floor_extracted(self) -> None: + r = SCRAPER.parse(_load(_STUDIO_ID), offer_url=_url(_STUDIO_ID)) + assert r is not None + assert r.floor == 9 + assert r.total_floors == 25 + + def test_studio_ceiling_kitchen_legitimately_absent(self) -> None: + # This is a newbuilding offer — Yandex does not expose ceiling/kitchen/ + # living for it. None here is legitimate, not a parse miss. + r = SCRAPER.parse(_load(_STUDIO_ID), offer_url=_url(_STUDIO_ID)) + assert r is not None + assert r.ceiling_height is None + assert r.kitchen_area_m2 is None + assert r.living_area_m2 is None + + +# --------------------------------------------------------------------------- +# Helper-level coverage +# --------------------------------------------------------------------------- + + +class TestStructuralHelpers: + def test_extract_offer_card_matches_offer_id(self) -> None: + card = _extract_offer_card(_load(_SECONDARY_ID), _SECONDARY_ID) + assert card is not None + assert card["offerId"] == _SECONDARY_ID + + def test_extract_offer_card_missing_state_returns_none(self) -> None: + assert _extract_offer_card("no state", "123") is None + + def test_parse_card_fields_space_objects(self) -> None: + card = { + "roomsTotal": 2, + "area": {"value": 55.3, "unit": "SQUARE_METER"}, + "livingSpace": {"value": 30.0, "unit": "SQUARE_METER"}, + "kitchenSpace": {"value": 12.5, "unit": "SQUARE_METER"}, + "ceilingHeight": 2.7, + "floorsTotal": 18, + "floorsOffered": [7], + "house": {"studio": False}, + } + rooms, area, living, kitchen, ceiling, floor, total = _parse_card_fields(card) + assert rooms == 2 + assert area == pytest.approx(55.3) + assert living == pytest.approx(30.0) + assert kitchen == pytest.approx(12.5) + assert ceiling == pytest.approx(2.7) + assert floor == 7 + assert total == 18 + + def test_parse_card_fields_studio_flag_sets_rooms_zero(self) -> None: + card = { + "roomsTotal": None, + "area": {"value": 24.92, "unit": "SQUARE_METER"}, + "floorsOffered": [9], + "floorsTotal": 25, + "house": {"studio": True}, + } + rooms, area, living, kitchen, ceiling, floor, total = _parse_card_fields(card) + assert rooms == 0 + assert area == pytest.approx(24.92) + assert living is None + assert kitchen is None + assert ceiling is None + assert floor == 9 + assert total == 25