227 lines
8.5 KiB
Python
227 lines
8.5 KiB
Python
"""Unit tests for YandexRealtyScraper DOM-based SERP parser."""
|
||
from datetime import date
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.yandex_realty import (
|
||
DEFAULT_CITY,
|
||
MAX_PAGES,
|
||
YandexRealtyScraper,
|
||
)
|
||
|
||
# Realistic single-card fixture (trimmed but selector-faithful).
|
||
# Note: selectolax.text(strip=True) concatenates all text nodes without
|
||
# added separators — so floor/price data lives in a single line div with
|
||
# middot (·) separators, matching real Yandex SERP card layout.
|
||
#
|
||
# The address container mirrors the live DOM shape (2026-05):
|
||
# <div class="AddressWithGeoLinks__addressContainer--XXXX">
|
||
# <a href=".../kupit/kvartira/st-...">улица Малышева</a>, 125
|
||
# </div>
|
||
# selectolax sees the trailing text node ", 125" as a sibling of <a>, so
|
||
# text(strip=False) yields "улица Малышева, 125" while text(strip=True)
|
||
# collapses whitespace to "улица Малышева,125".
|
||
SINGLE_CARD_HTML = """
|
||
<html><body>
|
||
<div data-test="OffersSerpItem">
|
||
<a href="/offer/7567094292504417257/">Открыть</a>
|
||
<div class="AddressWithGeoLinks__addressContainer--abc12">
|
||
<a href="/ekaterinburg/kupit/kvartira/st-frezerovshchikov-12345/">улица Фрезеровщиков</a>, 84
|
||
</div>
|
||
<a href="/ekaterinburg/kupit/novostrojka/tatlin-1592987/">ЖК Татлин</a>
|
||
<img src="https://avatars.mds.yandex.net/get-realty/12345/app_snippet_small/test.jpg">
|
||
<img src="https://avatars.mds.yandex.net/get-realty/67890/app_snippet_small/test2.jpg">
|
||
<img src="https://other-domain.example/photo.jpg">
|
||
<div>36,8 м² · 1-комнатная квартира · 8 этаж из 9 · 4 399 000 ₽ · 119 566 ₽ за м²</div>
|
||
<div>торг возможен · опубликовано 17 февраля 2026</div>
|
||
</div>
|
||
</body></html>
|
||
"""
|
||
|
||
STUDIO_CARD_HTML = """
|
||
<div data-test="OffersSerpItem">
|
||
<a href="/offer/9999999/">x</a>
|
||
<div>Студия · 28 м² · 2 этаж из 9 · 3 200 000 ₽</div>
|
||
</div>
|
||
"""
|
||
|
||
NO_PRICE_CARD_HTML = """
|
||
<div data-test="OffersSerpItem">
|
||
<a href="/offer/1111/">x</a>
|
||
<div>40 м² · 1-комнатная · 3 этаж из 5</div>
|
||
</div>
|
||
"""
|
||
|
||
# House number with letter suffix (e.g. "2В") — real-world variant
|
||
LETTER_HOUSE_CARD_HTML = """
|
||
<div data-test="OffersSerpItem">
|
||
<a href="/offer/2222/">x</a>
|
||
<div class="AddressWithGeoLinks__addressContainer--xyz">
|
||
<a href="/ekaterinburg/kupit/kvartira/st-ulica-pekhotintsev-1/">улица Пехотинцев</a>, 2В
|
||
</div>
|
||
<div>40 м² · 1-комнатная · 3 этаж из 5 · 5 000 000 ₽</div>
|
||
</div>
|
||
"""
|
||
|
||
# House number with slash (e.g. "14/3") — corner building
|
||
SLASH_HOUSE_CARD_HTML = """
|
||
<div data-test="OffersSerpItem">
|
||
<a href="/offer/3333/">x</a>
|
||
<div class="AddressWithGeoLinks__addressContainer--xyz">
|
||
<a href="/ekaterinburg/kupit/kvartira/st-akademika-landau-2/">улица Академика Ландау</a>, 14/3
|
||
</div>
|
||
<div>50 м² · 2-комнатная · 4 этаж из 9 · 7 000 000 ₽</div>
|
||
</div>
|
||
"""
|
||
|
||
# District / city prefix before the street link (cards in suburbs or
|
||
# named micro-districts) — must keep the full prefixed address.
|
||
DISTRICT_PREFIX_CARD_HTML = """
|
||
<div data-test="OffersSerpItem">
|
||
<a href="/offer/4444/">x</a>
|
||
<div class="AddressWithGeoLinks__addressContainer--xyz">
|
||
<a href="/berezovskiy/kupit/kvartira/">Берёзовский</a>,
|
||
<a href="/berezovskiy/kupit/kvartira/st-aprospekt-1/">Александровский проспект</a>, 5А
|
||
</div>
|
||
<div>35 м² · 1-комнатная · 2 этаж из 12 · 4 500 000 ₽</div>
|
||
</div>
|
||
"""
|
||
|
||
# Legacy fallback case: no AddressWithGeoLinks container — must still
|
||
# capture street name via the aggregator link (street-only) to avoid NULL.
|
||
NO_ADDR_CONTAINER_CARD_HTML = """
|
||
<div data-test="OffersSerpItem">
|
||
<a href="/offer/5555/">x</a>
|
||
<a href="/ekaterinburg/kupit/kvartira/st-ulica-malysheva-3/">улица Малышева</a>
|
||
<div>30 м² · 1-комнатная · 1 этаж из 5 · 3 500 000 ₽</div>
|
||
</div>
|
||
"""
|
||
|
||
EMPTY_PAGE_HTML = "<html><body><div>Нет объявлений</div></body></html>"
|
||
|
||
|
||
def test_max_pages_default():
|
||
assert MAX_PAGES == 3
|
||
|
||
|
||
def test_default_city():
|
||
assert DEFAULT_CITY == "ekaterinburg"
|
||
|
||
|
||
def test_build_url_first_page():
|
||
s = YandexRealtyScraper()
|
||
assert s._build_url(page=0) == (
|
||
"https://realty.yandex.ru/ekaterinburg/kupit/kvartira/vtorichniy-rynok/"
|
||
)
|
||
|
||
|
||
def test_build_url_paginated():
|
||
s = YandexRealtyScraper()
|
||
assert s._build_url(page=2) == (
|
||
"https://realty.yandex.ru/ekaterinburg/kupit/kvartira/vtorichniy-rynok/?page=2"
|
||
)
|
||
|
||
|
||
def test_build_url_other_city():
|
||
s = YandexRealtyScraper(city="moscow")
|
||
assert s._build_url(page=0) == (
|
||
"https://realty.yandex.ru/moscow/kupit/kvartira/vtorichniy-rynok/"
|
||
)
|
||
|
||
|
||
def test_parse_html_extracts_card():
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(SINGLE_CARD_HTML, page=0)
|
||
assert len(lots) == 1
|
||
lot = lots[0]
|
||
assert lot.source == "yandex"
|
||
assert lot.source_id == "7567094292504417257"
|
||
assert lot.source_url == "https://realty.yandex.ru/offer/7567094292504417257/"
|
||
# Address must include the house number (was "улица Фрезеровщиков" only — buggy)
|
||
assert lot.address == "улица Фрезеровщиков, 84"
|
||
assert any(ch.isdigit() for ch in lot.address)
|
||
assert lot.area_m2 == pytest.approx(36.8)
|
||
assert lot.rooms == 1
|
||
assert lot.floor == 8
|
||
assert lot.total_floors == 9
|
||
assert lot.price_rub == 4_399_000
|
||
assert lot.price_per_m2 == 119_566
|
||
assert lot.bargain_allowed is True
|
||
assert lot.listing_date == date(2026, 2, 17)
|
||
assert lot.house_source == "yandex_realty_nb"
|
||
assert lot.house_ext_id == "1592987"
|
||
assert lot.house_url == (
|
||
"https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/"
|
||
)
|
||
assert lot.listing_segment == "vtorichka"
|
||
assert lot.lat is None and lot.lon is None # SERP has no coords
|
||
# Photos: 2 from yandex CDN, the third (other-domain) skipped
|
||
assert len(lot.photo_urls) == 2
|
||
assert all("avatars.mds.yandex" in u for u in lot.photo_urls)
|
||
# Size upgrade applied: app_snippet_small → main
|
||
assert all("app_snippet_small" not in u for u in lot.photo_urls)
|
||
assert all("main" in u for u in lot.photo_urls)
|
||
|
||
|
||
def test_parse_html_studio_rooms_zero():
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(STUDIO_CARD_HTML, page=0)
|
||
assert len(lots) == 1
|
||
assert lots[0].rooms == 0 # studio convention
|
||
assert lots[0].area_m2 == pytest.approx(28.0)
|
||
assert lots[0].floor == 2 and lots[0].total_floors == 9
|
||
|
||
|
||
def test_parse_html_no_price_skipped():
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(NO_PRICE_CARD_HTML, page=0)
|
||
assert lots == []
|
||
|
||
|
||
def test_parse_html_empty_page():
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(EMPTY_PAGE_HTML, page=0)
|
||
assert lots == []
|
||
|
||
|
||
def test_parse_html_multi_card():
|
||
multi = SINGLE_CARD_HTML + STUDIO_CARD_HTML
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(multi, page=1)
|
||
assert len(lots) == 2
|
||
assert {lot.source_id for lot in lots} == {"7567094292504417257", "9999999"}
|
||
assert all(lot.raw_payload["page"] == 1 for lot in lots)
|
||
|
||
|
||
def test_address_with_letter_house_number():
|
||
"""House numbers with letter suffix (e.g. '2В') must round-trip intact."""
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(LETTER_HOUSE_CARD_HTML, page=0)
|
||
assert len(lots) == 1
|
||
assert lots[0].address == "улица Пехотинцев, 2В"
|
||
|
||
|
||
def test_address_with_slash_house_number():
|
||
"""Corner-building numbers like '14/3' must be preserved verbatim."""
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(SLASH_HOUSE_CARD_HTML, page=0)
|
||
assert len(lots) == 1
|
||
assert lots[0].address == "улица Академика Ландау, 14/3"
|
||
|
||
|
||
def test_address_keeps_district_prefix():
|
||
"""Cards with district / city prefix must keep the full prefixed
|
||
address — the geocoder benefits from disambiguation."""
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(DISTRICT_PREFIX_CARD_HTML, page=0)
|
||
assert len(lots) == 1
|
||
assert lots[0].address == "Берёзовский, Александровский проспект, 5А"
|
||
|
||
|
||
def test_address_fallback_to_street_link():
|
||
"""When the AddressWithGeoLinks container is missing, fall back to the
|
||
street aggregator link (street-only) — better than NULL."""
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(NO_ADDR_CONTAINER_CARD_HTML, page=0)
|
||
assert len(lots) == 1
|
||
assert lots[0].address == "улица Малышева"
|