fix(tradein): yandex_realty captures house number + parse_rub strips NBSP #593
4 changed files with 156 additions and 7 deletions
|
|
@ -281,13 +281,17 @@ def parse_rub(text: str | None) -> int | None:
|
|||
m = RE_RUB_MLN.search(text)
|
||||
if m:
|
||||
try:
|
||||
return int(float(m.group(1).replace(",", ".").replace(" ", "")) * 1_000_000)
|
||||
# `\s` strips also NBSP (\xa0), thin space, и т.д. — Yandex SERP
|
||||
# форматирует «9\xa0800\xa0000» через NBSP, не regular space.
|
||||
cleaned = re.sub(r"\s", "", m.group(1)).replace(",", ".")
|
||||
return int(float(cleaned) * 1_000_000)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
m = RE_RUB_RAW.search(text)
|
||||
if m:
|
||||
cleaned = m.group(1).replace(" ", "")
|
||||
# Same NBSP issue: `.replace(" ", "")` оставляет \xa0 → int() raises.
|
||||
cleaned = re.sub(r"\s", "", m.group(1))
|
||||
try:
|
||||
return int(cleaned)
|
||||
except ValueError:
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ Typical: 23 [data-test="OffersSerpItem"] cards per page.
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import date # noqa: F401 (used in type hints / future helpers)
|
||||
from typing import Any
|
||||
|
||||
|
|
@ -40,6 +41,11 @@ PHOTO_DOMAIN = "avatars.mds.yandex"
|
|||
PHOTO_SIZE_FROM = "app_snippet_small"
|
||||
PHOTO_SIZE_TO = "main"
|
||||
|
||||
# Address container wraps street link + house number text node.
|
||||
# Class is hashed but always starts with "AddressWithGeoLinks__addressContainer".
|
||||
ADDRESS_CONTAINER_SELECTOR = '[class*="AddressWithGeoLinks__addressContainer"]'
|
||||
_RE_WS = re.compile(r"\s+")
|
||||
|
||||
|
||||
class YandexRealtyScraper(BaseScraper):
|
||||
name = "yandex"
|
||||
|
|
@ -158,9 +164,10 @@ class YandexRealtyScraper(BaseScraper):
|
|||
bargain = "торг" in text.lower()
|
||||
listing_date = parse_listing_date(text)
|
||||
|
||||
# Address — street aggregator link
|
||||
street_link = card.css_first('a[href*="/kupit/kvartira/st-"]')
|
||||
address = street_link.text(strip=True) if street_link else None
|
||||
# Address — prefer the full address container (street + house number);
|
||||
# fall back to the street aggregator link (street name only) if absent.
|
||||
# See _extract_address for details.
|
||||
address = self._extract_address(card)
|
||||
|
||||
# Photos
|
||||
photo_urls = self._extract_photos(card)
|
||||
|
|
@ -211,6 +218,38 @@ class YandexRealtyScraper(BaseScraper):
|
|||
logger.exception("yandex card parse failed (page=%d)", page)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_address(card: Node) -> str | None:
|
||||
"""Extract full address (street + house number) from a SERP card.
|
||||
|
||||
Yandex renders the address inside a container like:
|
||||
<div class="AddressWithGeoLinks__addressContainer--XXXX">
|
||||
<a href="/.../kupit/kvartira/st-...">улица Энгельса</a>, 38
|
||||
</div>
|
||||
|
||||
The street-only `<a>` tag was used historically — it gave just the
|
||||
street name and broke forward-geocoding precision (street-level only,
|
||||
not exact house). Reading the container's text yields the full
|
||||
"<street>, <house>" pair, including letter/fraction suffixes
|
||||
(e.g. "2В", "2/2", "14к2"). Some cards prefix district/city info
|
||||
(e.g. "Берёзовский, Александровский проспект, 5А") — that's still
|
||||
a valid, more specific address for the geocoder.
|
||||
|
||||
Falls back to the street-only link when the container is missing
|
||||
(older layout / unexpected DOM) so we never regress to NULL.
|
||||
"""
|
||||
addr_div = card.css_first(ADDRESS_CONTAINER_SELECTOR)
|
||||
if addr_div is not None:
|
||||
# text(strip=False) preserves spaces between text nodes and
|
||||
# the anchor's text (selectolax with strip=True collapses them
|
||||
# and yields "улица Энгельса,38" instead of "улица Энгельса, 38").
|
||||
raw = addr_div.text(strip=False) or ""
|
||||
normalized = _RE_WS.sub(" ", raw).strip().strip(",").strip()
|
||||
if normalized:
|
||||
return normalized
|
||||
street_link = card.css_first('a[href*="/kupit/kvartira/st-"]')
|
||||
return street_link.text(strip=True) if street_link else None
|
||||
|
||||
@staticmethod
|
||||
def _parse_rooms(text: str) -> int | None:
|
||||
m = RE_TITLE_ROOMS.search(text)
|
||||
|
|
|
|||
|
|
@ -94,6 +94,21 @@ def test_parse_rub_empty():
|
|||
assert parse_rub("без цены") is None
|
||||
|
||||
|
||||
def test_parse_rub_nbsp_separator():
|
||||
"""Yandex SERP форматирует «9\\xa0800\\xa0000» через NBSP, не regular space.
|
||||
|
||||
До fix'а `.replace(" ", "")` оставлял \\xa0 → int() падал ValueError →
|
||||
parse_rub() возвращал None → весь scraper не парсил lots (0/23 на live HTML).
|
||||
"""
|
||||
assert parse_rub("9\xa0800\xa0000\xa0₽") == 9_800_000
|
||||
assert parse_rub("4\xa0399\xa0000") == 4_399_000
|
||||
|
||||
|
||||
def test_parse_rub_mln_with_nbsp():
|
||||
"""«4,4\\xa0млн\\xa0₽» — NBSP вокруг unit-keyword тоже встречается."""
|
||||
assert parse_rub("4,4\xa0млн\xa0₽") == 4_400_000
|
||||
|
||||
|
||||
# --- parse_ru_date ---
|
||||
def test_parse_ru_date_basic():
|
||||
assert parse_ru_date("9 мая 2026") == date(2026, 5, 9)
|
||||
|
|
|
|||
|
|
@ -13,11 +13,21 @@ from app.services.scrapers.yandex_realty import (
|
|||
# Note: selectolax.text(strip=True) concatenates all text nodes without
|
||||
# added separators — so floor/price data lives in a single line div with
|
||||
# middot (·) separators, matching real Yandex SERP card layout.
|
||||
#
|
||||
# The address container mirrors the live DOM shape (2026-05):
|
||||
# <div class="AddressWithGeoLinks__addressContainer--XXXX">
|
||||
# <a href=".../kupit/kvartira/st-...">улица Малышева</a>, 125
|
||||
# </div>
|
||||
# selectolax sees the trailing text node ", 125" as a sibling of <a>, so
|
||||
# text(strip=False) yields "улица Малышева, 125" while text(strip=True)
|
||||
# collapses whitespace to "улица Малышева,125".
|
||||
SINGLE_CARD_HTML = """
|
||||
<html><body>
|
||||
<div data-test="OffersSerpItem">
|
||||
<a href="/offer/7567094292504417257/">Открыть</a>
|
||||
<a href="/ekaterinburg/kupit/kvartira/st-frezerovshchikov-12345/">Фрезеровщиков</a>
|
||||
<div class="AddressWithGeoLinks__addressContainer--abc12">
|
||||
<a href="/ekaterinburg/kupit/kvartira/st-frezerovshchikov-12345/">улица Фрезеровщиков</a>, 84
|
||||
</div>
|
||||
<a href="/ekaterinburg/kupit/novostrojka/tatlin-1592987/">ЖК Татлин</a>
|
||||
<img src="https://avatars.mds.yandex.net/get-realty/12345/app_snippet_small/test.jpg">
|
||||
<img src="https://avatars.mds.yandex.net/get-realty/67890/app_snippet_small/test2.jpg">
|
||||
|
|
@ -42,6 +52,51 @@ NO_PRICE_CARD_HTML = """
|
|||
</div>
|
||||
"""
|
||||
|
||||
# House number with letter suffix (e.g. "2В") — real-world variant
|
||||
LETTER_HOUSE_CARD_HTML = """
|
||||
<div data-test="OffersSerpItem">
|
||||
<a href="/offer/2222/">x</a>
|
||||
<div class="AddressWithGeoLinks__addressContainer--xyz">
|
||||
<a href="/ekaterinburg/kupit/kvartira/st-ulica-pekhotintsev-1/">улица Пехотинцев</a>, 2В
|
||||
</div>
|
||||
<div>40 м² · 1-комнатная · 3 этаж из 5 · 5 000 000 ₽</div>
|
||||
</div>
|
||||
"""
|
||||
|
||||
# House number with slash (e.g. "14/3") — corner building
|
||||
SLASH_HOUSE_CARD_HTML = """
|
||||
<div data-test="OffersSerpItem">
|
||||
<a href="/offer/3333/">x</a>
|
||||
<div class="AddressWithGeoLinks__addressContainer--xyz">
|
||||
<a href="/ekaterinburg/kupit/kvartira/st-akademika-landau-2/">улица Академика Ландау</a>, 14/3
|
||||
</div>
|
||||
<div>50 м² · 2-комнатная · 4 этаж из 9 · 7 000 000 ₽</div>
|
||||
</div>
|
||||
"""
|
||||
|
||||
# District / city prefix before the street link (cards in suburbs or
|
||||
# named micro-districts) — must keep the full prefixed address.
|
||||
DISTRICT_PREFIX_CARD_HTML = """
|
||||
<div data-test="OffersSerpItem">
|
||||
<a href="/offer/4444/">x</a>
|
||||
<div class="AddressWithGeoLinks__addressContainer--xyz">
|
||||
<a href="/berezovskiy/kupit/kvartira/">Берёзовский</a>,
|
||||
<a href="/berezovskiy/kupit/kvartira/st-aprospekt-1/">Александровский проспект</a>, 5А
|
||||
</div>
|
||||
<div>35 м² · 1-комнатная · 2 этаж из 12 · 4 500 000 ₽</div>
|
||||
</div>
|
||||
"""
|
||||
|
||||
# Legacy fallback case: no AddressWithGeoLinks container — must still
|
||||
# capture street name via the aggregator link (street-only) to avoid NULL.
|
||||
NO_ADDR_CONTAINER_CARD_HTML = """
|
||||
<div data-test="OffersSerpItem">
|
||||
<a href="/offer/5555/">x</a>
|
||||
<a href="/ekaterinburg/kupit/kvartira/st-ulica-malysheva-3/">улица Малышева</a>
|
||||
<div>30 м² · 1-комнатная · 1 этаж из 5 · 3 500 000 ₽</div>
|
||||
</div>
|
||||
"""
|
||||
|
||||
EMPTY_PAGE_HTML = "<html><body><div>Нет объявлений</div></body></html>"
|
||||
|
||||
|
||||
|
|
@ -82,7 +137,9 @@ def test_parse_html_extracts_card():
|
|||
assert lot.source == "yandex"
|
||||
assert lot.source_id == "7567094292504417257"
|
||||
assert lot.source_url == "https://realty.yandex.ru/offer/7567094292504417257/"
|
||||
assert lot.address == "Фрезеровщиков"
|
||||
# Address must include the house number (was "улица Фрезеровщиков" only — buggy)
|
||||
assert lot.address == "улица Фрезеровщиков, 84"
|
||||
assert any(ch.isdigit() for ch in lot.address)
|
||||
assert lot.area_m2 == pytest.approx(36.8)
|
||||
assert lot.rooms == 1
|
||||
assert lot.floor == 8
|
||||
|
|
@ -134,3 +191,37 @@ def test_parse_html_multi_card():
|
|||
assert len(lots) == 2
|
||||
assert {lot.source_id for lot in lots} == {"7567094292504417257", "9999999"}
|
||||
assert all(lot.raw_payload["page"] == 1 for lot in lots)
|
||||
|
||||
|
||||
def test_address_with_letter_house_number():
|
||||
"""House numbers with letter suffix (e.g. '2В') must round-trip intact."""
|
||||
s = YandexRealtyScraper()
|
||||
lots = s._parse_html(LETTER_HOUSE_CARD_HTML, page=0)
|
||||
assert len(lots) == 1
|
||||
assert lots[0].address == "улица Пехотинцев, 2В"
|
||||
|
||||
|
||||
def test_address_with_slash_house_number():
|
||||
"""Corner-building numbers like '14/3' must be preserved verbatim."""
|
||||
s = YandexRealtyScraper()
|
||||
lots = s._parse_html(SLASH_HOUSE_CARD_HTML, page=0)
|
||||
assert len(lots) == 1
|
||||
assert lots[0].address == "улица Академика Ландау, 14/3"
|
||||
|
||||
|
||||
def test_address_keeps_district_prefix():
|
||||
"""Cards with district / city prefix must keep the full prefixed
|
||||
address — the geocoder benefits from disambiguation."""
|
||||
s = YandexRealtyScraper()
|
||||
lots = s._parse_html(DISTRICT_PREFIX_CARD_HTML, page=0)
|
||||
assert len(lots) == 1
|
||||
assert lots[0].address == "Берёзовский, Александровский проспект, 5А"
|
||||
|
||||
|
||||
def test_address_fallback_to_street_link():
|
||||
"""When the AddressWithGeoLinks container is missing, fall back to the
|
||||
street aggregator link (street-only) — better than NULL."""
|
||||
s = YandexRealtyScraper()
|
||||
lots = s._parse_html(NO_ADDR_CONTAINER_CARD_HTML, page=0)
|
||||
assert len(lots) == 1
|
||||
assert lots[0].address == "улица Малышева"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue