ead2ac16c5
Replace dead JSON-state extraction (Yandex moved to chunked/ephemeral state post-hydration) with selectolax DOM parsing using helpers from PR #456. Changes: - New URL: /{city}/kupit/kvartira/vtorichniy-rynok/?page=N (path-based, no geo) - Selector: [data-test="OffersSerpItem"] x ~23 per page - Per-card extraction: offer_id (URL regex), title (rooms/area/floor regex), price/ppm2 (regex + parse_rub), bargain flag, publish_date (parse_ru_date), street address, photos (avatars.mds.yandex domain filter + size upgrade), newbuilding linkage (house_source='yandex_realty_nb' + house_ext_id) - ScrapedLot mapping: lat/lon=None (no coords on SERP — geocode-missing fills later), listing_segment="vtorichka" - Pagination: fetch_around_multi_room -> up to MAX_PAGES=3 pages, dedup by source_id. **kwargs swallow keeps old rooms/sort/pages callers compat. - Preserved: BaseScraper interface, name="yandex" (matches existing rows + avoids breaking dedup_hash; Wave 5 conflict resolution can alias if needed), request_delay_sec=5.0, sleep_between_requests, tenacity retry. Tests: 10 unit tests against hand-crafted fixture HTML (single card / studio / no-price / empty page / multi-card / URL builder / city override) — all pass. Ruff clean.
136 lines
4.6 KiB
Python
136 lines
4.6 KiB
Python
"""Unit tests for YandexRealtyScraper DOM-based SERP parser."""
|
|
from datetime import date
|
|
|
|
import pytest
|
|
|
|
from app.services.scrapers.yandex_realty import (
|
|
DEFAULT_CITY,
|
|
MAX_PAGES,
|
|
YandexRealtyScraper,
|
|
)
|
|
|
|
# Realistic single-card fixture (trimmed but selector-faithful).
|
|
# Note: selectolax.text(strip=True) concatenates all text nodes without
|
|
# added separators — so floor/price data lives in a single line div with
|
|
# middot (·) separators, matching real Yandex SERP card layout.
|
|
SINGLE_CARD_HTML = """
|
|
<html><body>
|
|
<div data-test="OffersSerpItem">
|
|
<a href="/offer/7567094292504417257/">Открыть</a>
|
|
<a href="/ekaterinburg/kupit/kvartira/st-frezerovshchikov-12345/">Фрезеровщиков</a>
|
|
<a href="/ekaterinburg/kupit/novostrojka/tatlin-1592987/">ЖК Татлин</a>
|
|
<img src="https://avatars.mds.yandex.net/get-realty/12345/app_snippet_small/test.jpg">
|
|
<img src="https://avatars.mds.yandex.net/get-realty/67890/app_snippet_small/test2.jpg">
|
|
<img src="https://other-domain.example/photo.jpg">
|
|
<div>36,8 м² · 1-комнатная квартира · 8 этаж из 9 · 4 399 000 ₽ · 119 566 ₽ за м²</div>
|
|
<div>торг возможен · опубликовано 17 февраля 2026</div>
|
|
</div>
|
|
</body></html>
|
|
"""
|
|
|
|
STUDIO_CARD_HTML = """
|
|
<div data-test="OffersSerpItem">
|
|
<a href="/offer/9999999/">x</a>
|
|
<div>Студия · 28 м² · 2 этаж из 9 · 3 200 000 ₽</div>
|
|
</div>
|
|
"""
|
|
|
|
NO_PRICE_CARD_HTML = """
|
|
<div data-test="OffersSerpItem">
|
|
<a href="/offer/1111/">x</a>
|
|
<div>40 м² · 1-комнатная · 3 этаж из 5</div>
|
|
</div>
|
|
"""
|
|
|
|
EMPTY_PAGE_HTML = "<html><body><div>Нет объявлений</div></body></html>"
|
|
|
|
|
|
def test_max_pages_default():
|
|
assert MAX_PAGES == 3
|
|
|
|
|
|
def test_default_city():
|
|
assert DEFAULT_CITY == "ekaterinburg"
|
|
|
|
|
|
def test_build_url_first_page():
|
|
s = YandexRealtyScraper()
|
|
assert s._build_url(page=0) == (
|
|
"https://realty.yandex.ru/ekaterinburg/kupit/kvartira/vtorichniy-rynok/"
|
|
)
|
|
|
|
|
|
def test_build_url_paginated():
|
|
s = YandexRealtyScraper()
|
|
assert s._build_url(page=2) == (
|
|
"https://realty.yandex.ru/ekaterinburg/kupit/kvartira/vtorichniy-rynok/?page=2"
|
|
)
|
|
|
|
|
|
def test_build_url_other_city():
|
|
s = YandexRealtyScraper(city="moscow")
|
|
assert s._build_url(page=0) == (
|
|
"https://realty.yandex.ru/moscow/kupit/kvartira/vtorichniy-rynok/"
|
|
)
|
|
|
|
|
|
def test_parse_html_extracts_card():
|
|
s = YandexRealtyScraper()
|
|
lots = s._parse_html(SINGLE_CARD_HTML, page=0)
|
|
assert len(lots) == 1
|
|
lot = lots[0]
|
|
assert lot.source == "yandex"
|
|
assert lot.source_id == "7567094292504417257"
|
|
assert lot.source_url == "https://realty.yandex.ru/offer/7567094292504417257/"
|
|
assert lot.address == "Фрезеровщиков"
|
|
assert lot.area_m2 == pytest.approx(36.8)
|
|
assert lot.rooms == 1
|
|
assert lot.floor == 8
|
|
assert lot.total_floors == 9
|
|
assert lot.price_rub == 4_399_000
|
|
assert lot.price_per_m2 == 119_566
|
|
assert lot.bargain_allowed is True
|
|
assert lot.listing_date == date(2026, 2, 17)
|
|
assert lot.house_source == "yandex_realty_nb"
|
|
assert lot.house_ext_id == "1592987"
|
|
assert lot.house_url == (
|
|
"https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/"
|
|
)
|
|
assert lot.listing_segment == "vtorichka"
|
|
assert lot.lat is None and lot.lon is None # SERP has no coords
|
|
# Photos: 2 from yandex CDN, the third (other-domain) skipped
|
|
assert len(lot.photo_urls) == 2
|
|
assert all("avatars.mds.yandex" in u for u in lot.photo_urls)
|
|
# Size upgrade applied: app_snippet_small → main
|
|
assert all("app_snippet_small" not in u for u in lot.photo_urls)
|
|
assert all("main" in u for u in lot.photo_urls)
|
|
|
|
|
|
def test_parse_html_studio_rooms_zero():
|
|
s = YandexRealtyScraper()
|
|
lots = s._parse_html(STUDIO_CARD_HTML, page=0)
|
|
assert len(lots) == 1
|
|
assert lots[0].rooms == 0 # studio convention
|
|
assert lots[0].area_m2 == pytest.approx(28.0)
|
|
assert lots[0].floor == 2 and lots[0].total_floors == 9
|
|
|
|
|
|
def test_parse_html_no_price_skipped():
|
|
s = YandexRealtyScraper()
|
|
lots = s._parse_html(NO_PRICE_CARD_HTML, page=0)
|
|
assert lots == []
|
|
|
|
|
|
def test_parse_html_empty_page():
|
|
s = YandexRealtyScraper()
|
|
lots = s._parse_html(EMPTY_PAGE_HTML, page=0)
|
|
assert lots == []
|
|
|
|
|
|
def test_parse_html_multi_card():
|
|
multi = SINGLE_CARD_HTML + STUDIO_CARD_HTML
|
|
s = YandexRealtyScraper()
|
|
lots = s._parse_html(multi, page=1)
|
|
assert len(lots) == 2
|
|
assert {lot.source_id for lot in lots} == {"7567094292504417257", "9999999"}
|
|
assert all(lot.raw_payload["page"] == 1 for lot in lots)
|