From ead2ac16c5a6bf31719fbb348223fecd7a9ff6b8 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 23 May 2026 16:21:47 +0300 Subject: [PATCH] =?UTF-8?q?feat(tradein):=20yandex=5Frealty.py=20=E2=80=94?= =?UTF-8?q?=20DOM=20SERP=20refactor=20(vtorichka,=2023=20cards/page)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace dead JSON-state extraction (Yandex moved to chunked/ephemeral state post-hydration) with selectolax DOM parsing using helpers from PR #456. Changes: - New URL: /{city}/kupit/kvartira/vtorichniy-rynok/?page=N (path-based, no geo) - Selector: [data-test="OffersSerpItem"] x ~23 per page - Per-card extraction: offer_id (URL regex), title (rooms/area/floor regex), price/ppm2 (regex + parse_rub), bargain flag, publish_date (parse_ru_date), street address, photos (avatars.mds.yandex domain filter + size upgrade), newbuilding linkage (house_source='yandex_realty_nb' + house_ext_id) - ScrapedLot mapping: lat/lon=None (no coords on SERP — geocode-missing fills later), listing_segment="vtorichka" - Pagination: fetch_around_multi_room -> up to MAX_PAGES=3 pages, dedup by source_id. **kwargs swallow keeps old rooms/sort/pages callers compat. - Preserved: BaseScraper interface, name="yandex" (matches existing rows + avoids breaking dedup_hash; Wave 5 conflict resolution can alias if needed), request_delay_sec=5.0, sleep_between_requests, tenacity retry. Tests: 10 unit tests against hand-crafted fixture HTML (single card / studio / no-price / empty page / multi-card / URL builder / city override) — all pass. Ruff clean. --- .../app/services/scrapers/yandex_realty.py | 386 ++++++++---------- .../backend/tests/test_yandex_realty_serp.py | 136 ++++++ 2 files changed, 301 insertions(+), 221 deletions(-) create mode 100644 tradein-mvp/backend/tests/test_yandex_realty_serp.py diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py index c78ffa50..4f0cf654 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py @@ -1,290 +1,234 @@ -"""Yandex.Недвижимость scraper (realty.yandex.ru). +"""Yandex.Недвижимость scraper (realty.yandex.ru) — DOM-based SERP parser. -Я.Недвижимость отдаёт SSR HTML с `__INITIAL_STATE__` JSON в `', - re.DOTALL, -) - - -def _extract_yandex_state(html: str) -> dict[str, Any] | None: - m = _RE_STATE.search(html) - if not m: - return None - raw = m.group(1) - # Может начинаться с `window.__INITIAL_STATE__=` префикса - raw = re.sub(r'^\s*window\.[\w_]+\s*=\s*', '', raw).rstrip(';').strip() - try: - return json.loads(raw) - except json.JSONDecodeError: + @staticmethod + def _parse_rooms(text: str) -> int | None: + m = RE_TITLE_ROOMS.search(text) + if not m: + return None + if m.group(2): # studio + return 0 + if m.group(1): # numbered + try: + return int(m.group(1)) + except ValueError: + return None return None - -def _items_from_state(state: dict[str, Any]) -> list[dict[str, Any]]: - """Yandex Realty: state.map.offers.points содержит список объявлений (десктоп desktop SSR).""" - for path in [ - ("map", "offers", "points"), # верифицировано на проде - ("search", "offers"), - ("offers", "points"), - ("offers", "items"), - ]: - cursor: Any = state - for key in path: - if not isinstance(cursor, dict): - cursor = None + @staticmethod + def _extract_photos(card: Node) -> list[str]: + urls: list[str] = [] + for img in card.css("img"): + src = img.attributes.get("src", "") or "" + if PHOTO_DOMAIN in src: + urls.append(src.replace(PHOTO_SIZE_FROM, PHOTO_SIZE_TO)) + if len(urls) >= 5: break - cursor = cursor.get(key) - if isinstance(cursor, list) and cursor: - return cursor - return [] - - -def _map_wall_type(yandex_type: str | None) -> str | None: - if not yandex_type: - return None - mapping = { - "PANEL": "panel", - "BRICK": "brick", - "MONOLITH": "monolith", - "MONOLITH_BRICK": "monolith_brick", - "BLOCK": "panel", - } - return mapping.get(yandex_type.upper(), "other") + return urls diff --git a/tradein-mvp/backend/tests/test_yandex_realty_serp.py b/tradein-mvp/backend/tests/test_yandex_realty_serp.py new file mode 100644 index 00000000..e678bfec --- /dev/null +++ b/tradein-mvp/backend/tests/test_yandex_realty_serp.py @@ -0,0 +1,136 @@ +"""Unit tests for YandexRealtyScraper DOM-based SERP parser.""" +from datetime import date + +import pytest + +from app.services.scrapers.yandex_realty import ( + DEFAULT_CITY, + MAX_PAGES, + YandexRealtyScraper, +) + +# Realistic single-card fixture (trimmed but selector-faithful). +# Note: selectolax.text(strip=True) concatenates all text nodes without +# added separators — so floor/price data lives in a single line div with +# middot (·) separators, matching real Yandex SERP card layout. +SINGLE_CARD_HTML = """ + +
+ Открыть + Фрезеровщиков + ЖК Татлин + + + +
36,8 м² · 1-комнатная квартира · 8 этаж из 9 · 4 399 000 ₽ · 119 566 ₽ за м²
+
торг возможен · опубликовано 17 февраля 2026
+
+ +""" + +STUDIO_CARD_HTML = """ +
+ x +
Студия · 28 м² · 2 этаж из 9 · 3 200 000 ₽
+
+""" + +NO_PRICE_CARD_HTML = """ +
+ x +
40 м² · 1-комнатная · 3 этаж из 5
+
+""" + +EMPTY_PAGE_HTML = "
Нет объявлений
" + + +def test_max_pages_default(): + assert MAX_PAGES == 3 + + +def test_default_city(): + assert DEFAULT_CITY == "ekaterinburg" + + +def test_build_url_first_page(): + s = YandexRealtyScraper() + assert s._build_url(page=0) == ( + "https://realty.yandex.ru/ekaterinburg/kupit/kvartira/vtorichniy-rynok/" + ) + + +def test_build_url_paginated(): + s = YandexRealtyScraper() + assert s._build_url(page=2) == ( + "https://realty.yandex.ru/ekaterinburg/kupit/kvartira/vtorichniy-rynok/?page=2" + ) + + +def test_build_url_other_city(): + s = YandexRealtyScraper(city="moscow") + assert s._build_url(page=0) == ( + "https://realty.yandex.ru/moscow/kupit/kvartira/vtorichniy-rynok/" + ) + + +def test_parse_html_extracts_card(): + s = YandexRealtyScraper() + lots = s._parse_html(SINGLE_CARD_HTML, page=0) + assert len(lots) == 1 + lot = lots[0] + assert lot.source == "yandex" + assert lot.source_id == "7567094292504417257" + assert lot.source_url == "https://realty.yandex.ru/offer/7567094292504417257/" + assert lot.address == "Фрезеровщиков" + assert lot.area_m2 == pytest.approx(36.8) + assert lot.rooms == 1 + assert lot.floor == 8 + assert lot.total_floors == 9 + assert lot.price_rub == 4_399_000 + assert lot.price_per_m2 == 119_566 + assert lot.bargain_allowed is True + assert lot.listing_date == date(2026, 2, 17) + assert lot.house_source == "yandex_realty_nb" + assert lot.house_ext_id == "1592987" + assert lot.house_url == ( + "https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/" + ) + assert lot.listing_segment == "vtorichka" + assert lot.lat is None and lot.lon is None # SERP has no coords + # Photos: 2 from yandex CDN, the third (other-domain) skipped + assert len(lot.photo_urls) == 2 + assert all("avatars.mds.yandex" in u for u in lot.photo_urls) + # Size upgrade applied: app_snippet_small → main + assert all("app_snippet_small" not in u for u in lot.photo_urls) + assert all("main" in u for u in lot.photo_urls) + + +def test_parse_html_studio_rooms_zero(): + s = YandexRealtyScraper() + lots = s._parse_html(STUDIO_CARD_HTML, page=0) + assert len(lots) == 1 + assert lots[0].rooms == 0 # studio convention + assert lots[0].area_m2 == pytest.approx(28.0) + assert lots[0].floor == 2 and lots[0].total_floors == 9 + + +def test_parse_html_no_price_skipped(): + s = YandexRealtyScraper() + lots = s._parse_html(NO_PRICE_CARD_HTML, page=0) + assert lots == [] + + +def test_parse_html_empty_page(): + s = YandexRealtyScraper() + lots = s._parse_html(EMPTY_PAGE_HTML, page=0) + assert lots == [] + + +def test_parse_html_multi_card(): + multi = SINGLE_CARD_HTML + STUDIO_CARD_HTML + s = YandexRealtyScraper() + lots = s._parse_html(multi, page=1) + assert len(lots) == 2 + assert {lot.source_id for lot in lots} == {"7567094292504417257", "9999999"} + assert all(lot.raw_payload["page"] == 1 for lot in lots) -- 2.45.3