From aa70e9e155c261efd5c5d3a31cc6cf0706fe16f0 Mon Sep 17 00:00:00 2001 From: Light1YT Date: Wed, 27 May 2026 07:23:36 +0000 Subject: [PATCH] fix(tradein): yandex_realty captures house number + parse_rub strips NBSP (#593) --- .../app/services/scrapers/yandex_helpers.py | 8 +- .../app/services/scrapers/yandex_realty.py | 45 ++++++++- .../backend/tests/test_yandex_helpers.py | 15 +++ .../backend/tests/test_yandex_realty_serp.py | 95 ++++++++++++++++++- 4 files changed, 156 insertions(+), 7 deletions(-) diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_helpers.py b/tradein-mvp/backend/app/services/scrapers/yandex_helpers.py index 8b945943..573f6533 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_helpers.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_helpers.py @@ -281,13 +281,17 @@ def parse_rub(text: str | None) -> int | None: m = RE_RUB_MLN.search(text) if m: try: - return int(float(m.group(1).replace(",", ".").replace(" ", "")) * 1_000_000) + # `\s` strips also NBSP (\xa0), thin space, и т.д. — Yandex SERP + # форматирует «9\xa0800\xa0000» через NBSP, не regular space. + cleaned = re.sub(r"\s", "", m.group(1)).replace(",", ".") + return int(float(cleaned) * 1_000_000) except (ValueError, TypeError): pass m = RE_RUB_RAW.search(text) if m: - cleaned = m.group(1).replace(" ", "") + # Same NBSP issue: `.replace(" ", "")` оставляет \xa0 → int() raises. + cleaned = re.sub(r"\s", "", m.group(1)) try: return int(cleaned) except ValueError: diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py index f991dd81..203e2ce2 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py @@ -13,6 +13,7 @@ Typical: 23 [data-test="OffersSerpItem"] cards per page. from __future__ import annotations import logging +import re from datetime import date # noqa: F401 (used in type hints / future helpers) from typing import Any @@ -40,6 +41,11 @@ PHOTO_DOMAIN = "avatars.mds.yandex" PHOTO_SIZE_FROM = "app_snippet_small" PHOTO_SIZE_TO = "main" +# Address container wraps street link + house number text node. +# Class is hashed but always starts with "AddressWithGeoLinks__addressContainer". +ADDRESS_CONTAINER_SELECTOR = '[class*="AddressWithGeoLinks__addressContainer"]' +_RE_WS = re.compile(r"\s+") + class YandexRealtyScraper(BaseScraper): name = "yandex" @@ -158,9 +164,10 @@ class YandexRealtyScraper(BaseScraper): bargain = "торг" in text.lower() listing_date = parse_listing_date(text) - # Address — street aggregator link - street_link = card.css_first('a[href*="/kupit/kvartira/st-"]') - address = street_link.text(strip=True) if street_link else None + # Address — prefer the full address container (street + house number); + # fall back to the street aggregator link (street name only) if absent. + # See _extract_address for details. + address = self._extract_address(card) # Photos photo_urls = self._extract_photos(card) @@ -211,6 +218,38 @@ class YandexRealtyScraper(BaseScraper): logger.exception("yandex card parse failed (page=%d)", page) return None + @staticmethod + def _extract_address(card: Node) -> str | None: + """Extract full address (street + house number) from a SERP card. + + Yandex renders the address inside a container like: +
+ улица Энгельса, 38 +
+ + The street-only `` tag was used historically — it gave just the + street name and broke forward-geocoding precision (street-level only, + not exact house). Reading the container's text yields the full + ", " pair, including letter/fraction suffixes + (e.g. "2В", "2/2", "14к2"). Some cards prefix district/city info + (e.g. "Берёзовский, Александровский проспект, 5А") — that's still + a valid, more specific address for the geocoder. + + Falls back to the street-only link when the container is missing + (older layout / unexpected DOM) so we never regress to NULL. + """ + addr_div = card.css_first(ADDRESS_CONTAINER_SELECTOR) + if addr_div is not None: + # text(strip=False) preserves spaces between text nodes and + # the anchor's text (selectolax with strip=True collapses them + # and yields "улица Энгельса,38" instead of "улица Энгельса, 38"). + raw = addr_div.text(strip=False) or "" + normalized = _RE_WS.sub(" ", raw).strip().strip(",").strip() + if normalized: + return normalized + street_link = card.css_first('a[href*="/kupit/kvartira/st-"]') + return street_link.text(strip=True) if street_link else None + @staticmethod def _parse_rooms(text: str) -> int | None: m = RE_TITLE_ROOMS.search(text) diff --git a/tradein-mvp/backend/tests/test_yandex_helpers.py b/tradein-mvp/backend/tests/test_yandex_helpers.py index 21dc84ef..8f494c6f 100644 --- a/tradein-mvp/backend/tests/test_yandex_helpers.py +++ b/tradein-mvp/backend/tests/test_yandex_helpers.py @@ -94,6 +94,21 @@ def test_parse_rub_empty(): assert parse_rub("без цены") is None +def test_parse_rub_nbsp_separator(): + """Yandex SERP форматирует «9\\xa0800\\xa0000» через NBSP, не regular space. + + До fix'а `.replace(" ", "")` оставлял \\xa0 → int() падал ValueError → + parse_rub() возвращал None → весь scraper не парсил lots (0/23 на live HTML). + """ + assert parse_rub("9\xa0800\xa0000\xa0₽") == 9_800_000 + assert parse_rub("4\xa0399\xa0000") == 4_399_000 + + +def test_parse_rub_mln_with_nbsp(): + """«4,4\\xa0млн\\xa0₽» — NBSP вокруг unit-keyword тоже встречается.""" + assert parse_rub("4,4\xa0млн\xa0₽") == 4_400_000 + + # --- parse_ru_date --- def test_parse_ru_date_basic(): assert parse_ru_date("9 мая 2026") == date(2026, 5, 9) diff --git a/tradein-mvp/backend/tests/test_yandex_realty_serp.py b/tradein-mvp/backend/tests/test_yandex_realty_serp.py index e678bfec..1e48f035 100644 --- a/tradein-mvp/backend/tests/test_yandex_realty_serp.py +++ b/tradein-mvp/backend/tests/test_yandex_realty_serp.py @@ -13,11 +13,21 @@ from app.services.scrapers.yandex_realty import ( # Note: selectolax.text(strip=True) concatenates all text nodes without # added separators — so floor/price data lives in a single line div with # middot (·) separators, matching real Yandex SERP card layout. +# +# The address container mirrors the live DOM shape (2026-05): +#
+# улица Малышева, 125 +#
+# selectolax sees the trailing text node ", 125" as a sibling of , so +# text(strip=False) yields "улица Малышева, 125" while text(strip=True) +# collapses whitespace to "улица Малышева,125". SINGLE_CARD_HTML = """
Открыть - Фрезеровщиков +
+ улица Фрезеровщиков, 84 +
ЖК Татлин @@ -42,6 +52,51 @@ NO_PRICE_CARD_HTML = """
""" +# House number with letter suffix (e.g. "2В") — real-world variant +LETTER_HOUSE_CARD_HTML = """ +
+ x +
+ улица Пехотинцев, 2В +
+
40 м² · 1-комнатная · 3 этаж из 5 · 5 000 000 ₽
+
+""" + +# House number with slash (e.g. "14/3") — corner building +SLASH_HOUSE_CARD_HTML = """ +
+ x +
+ улица Академика Ландау, 14/3 +
+
50 м² · 2-комнатная · 4 этаж из 9 · 7 000 000 ₽
+
+""" + +# District / city prefix before the street link (cards in suburbs or +# named micro-districts) — must keep the full prefixed address. +DISTRICT_PREFIX_CARD_HTML = """ +
+ x +
+ Берёзовский, + Александровский проспект, 5А +
+
35 м² · 1-комнатная · 2 этаж из 12 · 4 500 000 ₽
+
+""" + +# Legacy fallback case: no AddressWithGeoLinks container — must still +# capture street name via the aggregator link (street-only) to avoid NULL. +NO_ADDR_CONTAINER_CARD_HTML = """ +
+ x + улица Малышева +
30 м² · 1-комнатная · 1 этаж из 5 · 3 500 000 ₽
+
+""" + EMPTY_PAGE_HTML = "
Нет объявлений
" @@ -82,7 +137,9 @@ def test_parse_html_extracts_card(): assert lot.source == "yandex" assert lot.source_id == "7567094292504417257" assert lot.source_url == "https://realty.yandex.ru/offer/7567094292504417257/" - assert lot.address == "Фрезеровщиков" + # Address must include the house number (was "улица Фрезеровщиков" only — buggy) + assert lot.address == "улица Фрезеровщиков, 84" + assert any(ch.isdigit() for ch in lot.address) assert lot.area_m2 == pytest.approx(36.8) assert lot.rooms == 1 assert lot.floor == 8 @@ -134,3 +191,37 @@ def test_parse_html_multi_card(): assert len(lots) == 2 assert {lot.source_id for lot in lots} == {"7567094292504417257", "9999999"} assert all(lot.raw_payload["page"] == 1 for lot in lots) + + +def test_address_with_letter_house_number(): + """House numbers with letter suffix (e.g. '2В') must round-trip intact.""" + s = YandexRealtyScraper() + lots = s._parse_html(LETTER_HOUSE_CARD_HTML, page=0) + assert len(lots) == 1 + assert lots[0].address == "улица Пехотинцев, 2В" + + +def test_address_with_slash_house_number(): + """Corner-building numbers like '14/3' must be preserved verbatim.""" + s = YandexRealtyScraper() + lots = s._parse_html(SLASH_HOUSE_CARD_HTML, page=0) + assert len(lots) == 1 + assert lots[0].address == "улица Академика Ландау, 14/3" + + +def test_address_keeps_district_prefix(): + """Cards with district / city prefix must keep the full prefixed + address — the geocoder benefits from disambiguation.""" + s = YandexRealtyScraper() + lots = s._parse_html(DISTRICT_PREFIX_CARD_HTML, page=0) + assert len(lots) == 1 + assert lots[0].address == "Берёзовский, Александровский проспект, 5А" + + +def test_address_fallback_to_street_link(): + """When the AddressWithGeoLinks container is missing, fall back to the + street aggregator link (street-only) — better than NULL.""" + s = YandexRealtyScraper() + lots = s._parse_html(NO_ADDR_CONTAINER_CARD_HTML, page=0) + assert len(lots) == 1 + assert lots[0].address == "улица Малышева"