bot-backend
d03c96a79a
Часть A (#24): yandex_realty.py парсил площадь из RAW-HTML (regex), пустого до JS-гидрации → 82% NULL area. _parse_html теперь извлекает {offer_id: area_m2} из <script id=initial_state_script> (или application/json) раз на страницу и использует preferentially в _card_to_lot; DOM-regex остаётся fallback'ом. 4 offer-path + 5 area-полей (incl nested {value}). 6 тестов. Часть B (#23, cian buildingCadastralNumber): УЖЕ реализовано — cian.py:291 читает offer.buildingCadastralNumber, :456 пишет в ScrapedLot, поле base.py:109 существует, тест test_cian_serp_cadastral_numbers зелёный. Изменений не нужно (премиса аудита устарела). cian.py НЕ трогался. ruff clean; pytest -k 'yandex or cian' 365 pass (+1 pre-existing unrelated fail test_cian_valuation low_price mock, не из этого PR).
341 lines
12 KiB
Python
341 lines
12 KiB
Python
"""Unit tests for YandexRealtyScraper DOM-based SERP parser."""
|
||
|
||
import json
|
||
from datetime import date
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.yandex_realty import (
|
||
DEFAULT_CITY,
|
||
MAX_PAGES,
|
||
YandexRealtyScraper,
|
||
_extract_offer_areas_from_state,
|
||
)
|
||
|
||
# Realistic single-card fixture (trimmed but selector-faithful).
|
||
# Note: selectolax.text(strip=True) concatenates all text nodes without
|
||
# added separators — so floor/price data lives in a single line div with
|
||
# middot (·) separators, matching real Yandex SERP card layout.
|
||
#
|
||
# The address container mirrors the live DOM shape (2026-05):
|
||
# <div class="AddressWithGeoLinks__addressContainer--XXXX">
|
||
# <a href=".../kupit/kvartira/st-...">улица Малышева</a>, 125
|
||
# </div>
|
||
# selectolax sees the trailing text node ", 125" as a sibling of <a>, so
|
||
# text(strip=False) yields "улица Малышева, 125" while text(strip=True)
|
||
# collapses whitespace to "улица Малышева,125".
|
||
SINGLE_CARD_HTML = """
|
||
<html><body>
|
||
<div data-test="OffersSerpItem">
|
||
<a href="/offer/7567094292504417257/">Открыть</a>
|
||
<div class="AddressWithGeoLinks__addressContainer--abc12">
|
||
<a href="/ekaterinburg/kupit/kvartira/st-frezerovshchikov-12345/">улица Фрезеровщиков</a>, 84
|
||
</div>
|
||
<a href="/ekaterinburg/kupit/novostrojka/tatlin-1592987/">ЖК Татлин</a>
|
||
<img src="https://avatars.mds.yandex.net/get-realty/12345/app_snippet_small/test.jpg">
|
||
<img src="https://avatars.mds.yandex.net/get-realty/67890/app_snippet_small/test2.jpg">
|
||
<img src="https://other-domain.example/photo.jpg">
|
||
<div>36,8 м² · 1-комнатная квартира · 8 этаж из 9 · 4 399 000 ₽ · 119 566 ₽ за м²</div>
|
||
<div>торг возможен · опубликовано 17 февраля 2026</div>
|
||
</div>
|
||
</body></html>
|
||
"""
|
||
|
||
STUDIO_CARD_HTML = """
|
||
<div data-test="OffersSerpItem">
|
||
<a href="/offer/9999999/">x</a>
|
||
<div>Студия · 28 м² · 2 этаж из 9 · 3 200 000 ₽</div>
|
||
</div>
|
||
"""
|
||
|
||
NO_PRICE_CARD_HTML = """
|
||
<div data-test="OffersSerpItem">
|
||
<a href="/offer/1111/">x</a>
|
||
<div>40 м² · 1-комнатная · 3 этаж из 5</div>
|
||
</div>
|
||
"""
|
||
|
||
# House number with letter suffix (e.g. "2В") — real-world variant
|
||
LETTER_HOUSE_CARD_HTML = """
|
||
<div data-test="OffersSerpItem">
|
||
<a href="/offer/2222/">x</a>
|
||
<div class="AddressWithGeoLinks__addressContainer--xyz">
|
||
<a href="/ekaterinburg/kupit/kvartira/st-ulica-pekhotintsev-1/">улица Пехотинцев</a>, 2В
|
||
</div>
|
||
<div>40 м² · 1-комнатная · 3 этаж из 5 · 5 000 000 ₽</div>
|
||
</div>
|
||
"""
|
||
|
||
# House number with slash (e.g. "14/3") — corner building
|
||
SLASH_HOUSE_CARD_HTML = """
|
||
<div data-test="OffersSerpItem">
|
||
<a href="/offer/3333/">x</a>
|
||
<div class="AddressWithGeoLinks__addressContainer--xyz">
|
||
<a href="/ekaterinburg/kupit/kvartira/st-akademika-landau-2/">улица Академика Ландау</a>, 14/3
|
||
</div>
|
||
<div>50 м² · 2-комнатная · 4 этаж из 9 · 7 000 000 ₽</div>
|
||
</div>
|
||
"""
|
||
|
||
# District / city prefix before the street link (cards in suburbs or
|
||
# named micro-districts) — must keep the full prefixed address.
|
||
DISTRICT_PREFIX_CARD_HTML = """
|
||
<div data-test="OffersSerpItem">
|
||
<a href="/offer/4444/">x</a>
|
||
<div class="AddressWithGeoLinks__addressContainer--xyz">
|
||
<a href="/berezovskiy/kupit/kvartira/">Берёзовский</a>,
|
||
<a href="/berezovskiy/kupit/kvartira/st-aprospekt-1/">Александровский проспект</a>, 5А
|
||
</div>
|
||
<div>35 м² · 1-комнатная · 2 этаж из 12 · 4 500 000 ₽</div>
|
||
</div>
|
||
"""
|
||
|
||
# Legacy fallback case: no AddressWithGeoLinks container — must still
|
||
# capture street name via the aggregator link (street-only) to avoid NULL.
|
||
NO_ADDR_CONTAINER_CARD_HTML = """
|
||
<div data-test="OffersSerpItem">
|
||
<a href="/offer/5555/">x</a>
|
||
<a href="/ekaterinburg/kupit/kvartira/st-ulica-malysheva-3/">улица Малышева</a>
|
||
<div>30 м² · 1-комнатная · 1 этаж из 5 · 3 500 000 ₽</div>
|
||
</div>
|
||
"""
|
||
|
||
EMPTY_PAGE_HTML = "<html><body><div>Нет объявлений</div></body></html>"
|
||
|
||
|
||
def test_max_pages_default():
|
||
assert MAX_PAGES == 3
|
||
|
||
|
||
def test_default_city():
|
||
assert DEFAULT_CITY == "ekaterinburg"
|
||
|
||
|
||
def test_build_url_first_page():
|
||
s = YandexRealtyScraper()
|
||
assert s._build_url(page=0) == (
|
||
"https://realty.yandex.ru/ekaterinburg/kupit/kvartira/vtorichniy-rynok/"
|
||
)
|
||
|
||
|
||
def test_build_url_paginated():
|
||
s = YandexRealtyScraper()
|
||
assert s._build_url(page=2) == (
|
||
"https://realty.yandex.ru/ekaterinburg/kupit/kvartira/vtorichniy-rynok/?page=2"
|
||
)
|
||
|
||
|
||
def test_build_url_other_city():
|
||
s = YandexRealtyScraper(city="moscow")
|
||
assert s._build_url(page=0) == (
|
||
"https://realty.yandex.ru/moscow/kupit/kvartira/vtorichniy-rynok/"
|
||
)
|
||
|
||
|
||
def test_parse_html_extracts_card():
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(SINGLE_CARD_HTML, page=0)
|
||
assert len(lots) == 1
|
||
lot = lots[0]
|
||
assert lot.source == "yandex"
|
||
assert lot.source_id == "7567094292504417257"
|
||
assert lot.source_url == "https://realty.yandex.ru/offer/7567094292504417257/"
|
||
# Address must include the house number (was "улица Фрезеровщиков" only — buggy)
|
||
assert lot.address == "улица Фрезеровщиков, 84"
|
||
assert any(ch.isdigit() for ch in lot.address)
|
||
assert lot.area_m2 == pytest.approx(36.8)
|
||
assert lot.rooms == 1
|
||
assert lot.floor == 8
|
||
assert lot.total_floors == 9
|
||
assert lot.price_rub == 4_399_000
|
||
assert lot.price_per_m2 == 119_566
|
||
assert lot.bargain_allowed is True
|
||
assert lot.listing_date == date(2026, 2, 17)
|
||
assert lot.house_source == "yandex_realty_nb"
|
||
assert lot.house_ext_id == "1592987"
|
||
assert lot.house_url == (
|
||
"https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/"
|
||
)
|
||
assert lot.listing_segment == "vtorichka"
|
||
assert lot.lat is None and lot.lon is None # SERP has no coords
|
||
# Photos: 2 from yandex CDN, the third (other-domain) skipped
|
||
assert len(lot.photo_urls) == 2
|
||
assert all("avatars.mds.yandex" in u for u in lot.photo_urls)
|
||
# Size upgrade applied: app_snippet_small → main
|
||
assert all("app_snippet_small" not in u for u in lot.photo_urls)
|
||
assert all("main" in u for u in lot.photo_urls)
|
||
|
||
|
||
def test_parse_html_studio_rooms_zero():
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(STUDIO_CARD_HTML, page=0)
|
||
assert len(lots) == 1
|
||
assert lots[0].rooms == 0 # studio convention
|
||
assert lots[0].area_m2 == pytest.approx(28.0)
|
||
assert lots[0].floor == 2 and lots[0].total_floors == 9
|
||
|
||
|
||
def test_parse_html_no_price_skipped():
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(NO_PRICE_CARD_HTML, page=0)
|
||
assert lots == []
|
||
|
||
|
||
def test_parse_html_empty_page():
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(EMPTY_PAGE_HTML, page=0)
|
||
assert lots == []
|
||
|
||
|
||
def test_parse_html_multi_card():
|
||
multi = SINGLE_CARD_HTML + STUDIO_CARD_HTML
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(multi, page=1)
|
||
assert len(lots) == 2
|
||
assert {lot.source_id for lot in lots} == {"7567094292504417257", "9999999"}
|
||
assert all(lot.raw_payload["page"] == 1 for lot in lots)
|
||
|
||
|
||
def test_address_with_letter_house_number():
|
||
"""House numbers with letter suffix (e.g. '2В') must round-trip intact."""
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(LETTER_HOUSE_CARD_HTML, page=0)
|
||
assert len(lots) == 1
|
||
assert lots[0].address == "улица Пехотинцев, 2В"
|
||
|
||
|
||
def test_address_with_slash_house_number():
|
||
"""Corner-building numbers like '14/3' must be preserved verbatim."""
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(SLASH_HOUSE_CARD_HTML, page=0)
|
||
assert len(lots) == 1
|
||
assert lots[0].address == "улица Академика Ландау, 14/3"
|
||
|
||
|
||
def test_address_keeps_district_prefix():
|
||
"""Cards with district / city prefix must keep the full prefixed
|
||
address — the geocoder benefits from disambiguation."""
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(DISTRICT_PREFIX_CARD_HTML, page=0)
|
||
assert len(lots) == 1
|
||
assert lots[0].address == "Берёзовский, Александровский проспект, 5А"
|
||
|
||
|
||
def test_address_fallback_to_street_link():
|
||
"""When the AddressWithGeoLinks container is missing, fall back to the
|
||
street aggregator link (street-only) — better than NULL."""
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(NO_ADDR_CONTAINER_CARD_HTML, page=0)
|
||
assert len(lots) == 1
|
||
assert lots[0].address == "улица Малышева"
|
||
|
||
|
||
# ── PART A: area from embedded JSON state ────────────────────────────────────
|
||
|
||
|
||
def _make_yandex_state_html(offer_id: str, area: float, price_rub: int = 5_000_000) -> str:
|
||
"""Build minimal Yandex SERP HTML with:
|
||
- <script id="initial_state_script"> containing offer area in JSON state
|
||
- A DOM card where the text div has NO area (simulating pre-JS-hydration).
|
||
"""
|
||
state = {
|
||
"offers": [
|
||
{
|
||
"offerId": offer_id,
|
||
"totalArea": area,
|
||
}
|
||
]
|
||
}
|
||
state_json = json.dumps(state, ensure_ascii=False)
|
||
price_fmt = f"{price_rub:,}".replace(",", " ")
|
||
return f"""<html><head>
|
||
<script id="initial_state_script">{state_json}</script>
|
||
</head><body>
|
||
<div data-test="OffersSerpItem">
|
||
<a href="/offer/{offer_id}/">Открыть</a>
|
||
<div class="AddressWithGeoLinks__addressContainer--xyz">
|
||
<a href="/ekaterinburg/kupit/kvartira/st-test-1/">улица Тестовая</a>, 10
|
||
</div>
|
||
<div>{price_fmt} ₽</div>
|
||
</div>
|
||
</body></html>"""
|
||
|
||
|
||
def test_area_from_state_json_no_dom_text():
|
||
"""area_m2 extracted from embedded JSON state when DOM text has no area."""
|
||
html = _make_yandex_state_html(offer_id="1234567890", area=63.5)
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(html, page=0)
|
||
assert len(lots) == 1
|
||
assert lots[0].area_m2 == pytest.approx(63.5)
|
||
|
||
|
||
def test_area_from_state_json_multiple_offers():
|
||
"""State with multiple offers — each card gets its own area from state."""
|
||
state = {
|
||
"offers": [
|
||
{"offerId": "111", "totalArea": 42.0},
|
||
{"offerId": "222", "totalArea": 78.3},
|
||
]
|
||
}
|
||
state_json = json.dumps(state)
|
||
html = f"""<html><head>
|
||
<script id="initial_state_script">{state_json}</script>
|
||
</head><body>
|
||
<div data-test="OffersSerpItem">
|
||
<a href="/offer/111/">x</a>
|
||
<div>3 000 000 ₽</div>
|
||
</div>
|
||
<div data-test="OffersSerpItem">
|
||
<a href="/offer/222/">x</a>
|
||
<div>5 000 000 ₽</div>
|
||
</div>
|
||
</body></html>"""
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(html, page=0)
|
||
assert len(lots) == 2
|
||
areas = {lot.source_id: lot.area_m2 for lot in lots}
|
||
assert areas["111"] == pytest.approx(42.0)
|
||
assert areas["222"] == pytest.approx(78.3)
|
||
|
||
|
||
def test_area_dom_fallback_when_no_state():
|
||
"""When state script absent, area falls back to DOM regex (existing behavior)."""
|
||
# SINGLE_CARD_HTML has area in DOM text — state script absent
|
||
s = YandexRealtyScraper()
|
||
lots = s._parse_html(SINGLE_CARD_HTML, page=0)
|
||
assert len(lots) == 1
|
||
assert lots[0].area_m2 == pytest.approx(36.8)
|
||
|
||
|
||
def test_extract_offer_areas_from_state_nested_path():
|
||
"""State via pageData.offers path (SERP v2 shape)."""
|
||
state = {
|
||
"pageData": {
|
||
"offers": [
|
||
{"offerId": "999", "spaceTotal": 55.0},
|
||
]
|
||
}
|
||
}
|
||
html = '<script id="initial_state_script">' + json.dumps(state) + "</script>"
|
||
areas = _extract_offer_areas_from_state(html)
|
||
assert areas == {"999": 55.0}
|
||
|
||
|
||
def test_extract_offer_areas_area_dict_value():
|
||
"""area field as nested dict {'value': 48.0, 'unit': 'SQM'} (some Yandex responses)."""
|
||
state = {
|
||
"offers": [
|
||
{"offerId": "777", "area": {"value": 48.0, "unit": "SQM"}},
|
||
]
|
||
}
|
||
html = '<script id="initial_state_script">' + json.dumps(state) + "</script>"
|
||
areas = _extract_offer_areas_from_state(html)
|
||
assert areas == {"777": 48.0}
|
||
|
||
|
||
def test_extract_offer_areas_empty_when_no_script():
|
||
"""No state script → empty dict returned, no crash."""
|
||
html = "<html><body><p>no state here</p></body></html>"
|
||
areas = _extract_offer_areas_from_state(html)
|
||
assert areas == {}
|