fix(tradein): yandex_realty captures house number + parse_rub strips NBSP #593

Merged
Light1YT merged 1 commit from fix/yandex-realty-address-with-house-number into main 2026-05-27 07:23:36 +00:00
4 changed files with 156 additions and 7 deletions

View file

@ -281,13 +281,17 @@ def parse_rub(text: str | None) -> int | None:
m = RE_RUB_MLN.search(text)
if m:
try:
return int(float(m.group(1).replace(",", ".").replace(" ", "")) * 1_000_000)
# `\s` strips also NBSP (\xa0), thin space, и т.д. — Yandex SERP
# форматирует «9\xa0800\xa0000» через NBSP, не regular space.
cleaned = re.sub(r"\s", "", m.group(1)).replace(",", ".")
return int(float(cleaned) * 1_000_000)
except (ValueError, TypeError):
pass
m = RE_RUB_RAW.search(text)
if m:
cleaned = m.group(1).replace(" ", "")
# Same NBSP issue: `.replace(" ", "")` оставляет \xa0 → int() raises.
cleaned = re.sub(r"\s", "", m.group(1))
try:
return int(cleaned)
except ValueError:

View file

@ -13,6 +13,7 @@ Typical: 23 [data-test="OffersSerpItem"] cards per page.
from __future__ import annotations
import logging
import re
from datetime import date # noqa: F401 (used in type hints / future helpers)
from typing import Any
@ -40,6 +41,11 @@ PHOTO_DOMAIN = "avatars.mds.yandex"
PHOTO_SIZE_FROM = "app_snippet_small"
PHOTO_SIZE_TO = "main"
# Address container wraps street link + house number text node.
# Class is hashed but always starts with "AddressWithGeoLinks__addressContainer".
ADDRESS_CONTAINER_SELECTOR = '[class*="AddressWithGeoLinks__addressContainer"]'
_RE_WS = re.compile(r"\s+")
class YandexRealtyScraper(BaseScraper):
name = "yandex"
@ -158,9 +164,10 @@ class YandexRealtyScraper(BaseScraper):
bargain = "торг" in text.lower()
listing_date = parse_listing_date(text)
# Address — street aggregator link
street_link = card.css_first('a[href*="/kupit/kvartira/st-"]')
address = street_link.text(strip=True) if street_link else None
# Address — prefer the full address container (street + house number);
# fall back to the street aggregator link (street name only) if absent.
# See _extract_address for details.
address = self._extract_address(card)
# Photos
photo_urls = self._extract_photos(card)
@ -211,6 +218,38 @@ class YandexRealtyScraper(BaseScraper):
logger.exception("yandex card parse failed (page=%d)", page)
return None
@staticmethod
def _extract_address(card: Node) -> str | None:
"""Extract full address (street + house number) from a SERP card.
Yandex renders the address inside a container like:
<div class="AddressWithGeoLinks__addressContainer--XXXX">
<a href="/.../kupit/kvartira/st-...">улица Энгельса</a>, 38
</div>
The street-only `<a>` tag was used historically it gave just the
street name and broke forward-geocoding precision (street-level only,
not exact house). Reading the container's text yields the full
"<street>, <house>" pair, including letter/fraction suffixes
(e.g. "2В", "2/2", "14к2"). Some cards prefix district/city info
(e.g. "Берёзовский, Александровский проспект, 5А") that's still
a valid, more specific address for the geocoder.
Falls back to the street-only link when the container is missing
(older layout / unexpected DOM) so we never regress to NULL.
"""
addr_div = card.css_first(ADDRESS_CONTAINER_SELECTOR)
if addr_div is not None:
# text(strip=False) preserves spaces between text nodes and
# the anchor's text (selectolax with strip=True collapses them
# and yields "улица Энгельса,38" instead of "улица Энгельса, 38").
raw = addr_div.text(strip=False) or ""
normalized = _RE_WS.sub(" ", raw).strip().strip(",").strip()
if normalized:
return normalized
street_link = card.css_first('a[href*="/kupit/kvartira/st-"]')
return street_link.text(strip=True) if street_link else None
@staticmethod
def _parse_rooms(text: str) -> int | None:
m = RE_TITLE_ROOMS.search(text)

View file

@ -94,6 +94,21 @@ def test_parse_rub_empty():
assert parse_rub("без цены") is None
def test_parse_rub_nbsp_separator():
"""Yandex SERP форматирует «9\\xa0800\\xa0000» через NBSP, не regular space.
До fix'а `.replace(" ", "")` оставлял \\xa0 → int() падал ValueError →
parse_rub() возвращал None весь scraper не парсил lots (0/23 на live HTML).
"""
assert parse_rub("9\xa0800\xa0000\xa0") == 9_800_000
assert parse_rub("4\xa0399\xa0000") == 4_399_000
def test_parse_rub_mln_with_nbsp():
"""«4,4\\xa0млн\\xa0₽» — NBSP вокруг unit-keyword тоже встречается."""
assert parse_rub("4,4\xa0млн\xa0") == 4_400_000
# --- parse_ru_date ---
def test_parse_ru_date_basic():
assert parse_ru_date("9 мая 2026") == date(2026, 5, 9)

View file

@ -13,11 +13,21 @@ from app.services.scrapers.yandex_realty import (
# Note: selectolax.text(strip=True) concatenates all text nodes without
# added separators — so floor/price data lives in a single line div with
# middot (·) separators, matching real Yandex SERP card layout.
#
# The address container mirrors the live DOM shape (2026-05):
# <div class="AddressWithGeoLinks__addressContainer--XXXX">
# <a href=".../kupit/kvartira/st-...">улица Малышева</a>, 125
# </div>
# selectolax sees the trailing text node ", 125" as a sibling of <a>, so
# text(strip=False) yields "улица Малышева, 125" while text(strip=True)
# collapses whitespace to "улица Малышева,125".
SINGLE_CARD_HTML = """
<html><body>
<div data-test="OffersSerpItem">
<a href="/offer/7567094292504417257/">Открыть</a>
<a href="/ekaterinburg/kupit/kvartira/st-frezerovshchikov-12345/">Фрезеровщиков</a>
<div class="AddressWithGeoLinks__addressContainer--abc12">
<a href="/ekaterinburg/kupit/kvartira/st-frezerovshchikov-12345/">улица Фрезеровщиков</a>, 84
</div>
<a href="/ekaterinburg/kupit/novostrojka/tatlin-1592987/">ЖК Татлин</a>
<img src="https://avatars.mds.yandex.net/get-realty/12345/app_snippet_small/test.jpg">
<img src="https://avatars.mds.yandex.net/get-realty/67890/app_snippet_small/test2.jpg">
@ -42,6 +52,51 @@ NO_PRICE_CARD_HTML = """
</div>
"""
# House number with letter suffix (e.g. "2В") — real-world variant
LETTER_HOUSE_CARD_HTML = """
<div data-test="OffersSerpItem">
<a href="/offer/2222/">x</a>
<div class="AddressWithGeoLinks__addressContainer--xyz">
<a href="/ekaterinburg/kupit/kvartira/st-ulica-pekhotintsev-1/">улица Пехотинцев</a>, 2В
</div>
<div>40 м² · 1-комнатная · 3 этаж из 5 · 5 000 000 </div>
</div>
"""
# House number with slash (e.g. "14/3") — corner building
SLASH_HOUSE_CARD_HTML = """
<div data-test="OffersSerpItem">
<a href="/offer/3333/">x</a>
<div class="AddressWithGeoLinks__addressContainer--xyz">
<a href="/ekaterinburg/kupit/kvartira/st-akademika-landau-2/">улица Академика Ландау</a>, 14/3
</div>
<div>50 м² · 2-комнатная · 4 этаж из 9 · 7 000 000 </div>
</div>
"""
# District / city prefix before the street link (cards in suburbs or
# named micro-districts) — must keep the full prefixed address.
DISTRICT_PREFIX_CARD_HTML = """
<div data-test="OffersSerpItem">
<a href="/offer/4444/">x</a>
<div class="AddressWithGeoLinks__addressContainer--xyz">
<a href="/berezovskiy/kupit/kvartira/">Берёзовский</a>,
<a href="/berezovskiy/kupit/kvartira/st-aprospekt-1/">Александровский проспект</a>, 5А
</div>
<div>35 м² · 1-комнатная · 2 этаж из 12 · 4 500 000 </div>
</div>
"""
# Legacy fallback case: no AddressWithGeoLinks container — must still
# capture street name via the aggregator link (street-only) to avoid NULL.
NO_ADDR_CONTAINER_CARD_HTML = """
<div data-test="OffersSerpItem">
<a href="/offer/5555/">x</a>
<a href="/ekaterinburg/kupit/kvartira/st-ulica-malysheva-3/">улица Малышева</a>
<div>30 м² · 1-комнатная · 1 этаж из 5 · 3 500 000 </div>
</div>
"""
EMPTY_PAGE_HTML = "<html><body><div>Нет объявлений</div></body></html>"
@ -82,7 +137,9 @@ def test_parse_html_extracts_card():
assert lot.source == "yandex"
assert lot.source_id == "7567094292504417257"
assert lot.source_url == "https://realty.yandex.ru/offer/7567094292504417257/"
assert lot.address == "Фрезеровщиков"
# Address must include the house number (was "улица Фрезеровщиков" only — buggy)
assert lot.address == "улица Фрезеровщиков, 84"
assert any(ch.isdigit() for ch in lot.address)
assert lot.area_m2 == pytest.approx(36.8)
assert lot.rooms == 1
assert lot.floor == 8
@ -134,3 +191,37 @@ def test_parse_html_multi_card():
assert len(lots) == 2
assert {lot.source_id for lot in lots} == {"7567094292504417257", "9999999"}
assert all(lot.raw_payload["page"] == 1 for lot in lots)
def test_address_with_letter_house_number():
"""House numbers with letter suffix (e.g. '2В') must round-trip intact."""
s = YandexRealtyScraper()
lots = s._parse_html(LETTER_HOUSE_CARD_HTML, page=0)
assert len(lots) == 1
assert lots[0].address == "улица Пехотинцев, 2В"
def test_address_with_slash_house_number():
"""Corner-building numbers like '14/3' must be preserved verbatim."""
s = YandexRealtyScraper()
lots = s._parse_html(SLASH_HOUSE_CARD_HTML, page=0)
assert len(lots) == 1
assert lots[0].address == "улица Академика Ландау, 14/3"
def test_address_keeps_district_prefix():
"""Cards with district / city prefix must keep the full prefixed
address the geocoder benefits from disambiguation."""
s = YandexRealtyScraper()
lots = s._parse_html(DISTRICT_PREFIX_CARD_HTML, page=0)
assert len(lots) == 1
assert lots[0].address == "Берёзовский, Александровский проспект, 5А"
def test_address_fallback_to_street_link():
"""When the AddressWithGeoLinks container is missing, fall back to the
street aggregator link (street-only) better than NULL."""
s = YandexRealtyScraper()
lots = s._parse_html(NO_ADDR_CONTAINER_CARD_HTML, page=0)
assert len(lots) == 1
assert lots[0].address == "улица Малышева"