bot-backend
278dafba76
All checks were successful
CI / changes (push) Successful in 8s
CI / backend-tests (push) Has been skipped
CI / frontend-tests (push) Has been skipped
CI / openapi-codegen-check (push) Has been skipped
CI / changes (pull_request) Successful in 6s
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
Валидационный yandex_newbuilding_sweep (run 80) заполнял ratings_count / text_reviews_count / developer_name / lat-lon, но rating оставался NULL для всех ЖК. Live-диагностика (прод-браузер, ЖК Татлин/Рио 2026-06-15): рейтинг рендерится как «4.3из 5» / «4,5из 5» — БЕЗ пробела перед «из» (nbsp внутри «из 5»), а RE_RATING требовал \s+ перед «из» → match=None. - RE_RATING: (\d[.,]\d)\s+из\s+5 → (\d[.,]\d)\s*из\s+5 (пробел перед «из» опционален). \s уже матчит nbsp, поэтому «из\xa05» обрабатывался; ломал именно отсутствующий разделитель перед «из». - test_parse_rating_no_space_before_iz: регрессия на обе дрейфовые формы. Существующие rating-тесты («4,3 из 5» с пробелом) остаются зелёными (\s* ⊇ \s+). Refs #974
441 lines
16 KiB
Python
441 lines
16 KiB
Python
"""Unit tests for yandex_newbuilding.py — ЖК landing parser (Worker B, Wave 4).
|
||
|
||
Reference target: ЖК Татлин (slug=tatlin, id=1592987) — comfort+, June 2023,
|
||
PRINZIP developer, rating 4.3, 1505 ratings, 353 text reviews,
|
||
coords (56.855312, 60.576668).
|
||
|
||
All tests are offline — no network calls. Parser receives hand-crafted fixture HTML.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.yandex_newbuilding import (
|
||
JKMetroStation,
|
||
YandexNewbuildingInfo,
|
||
YandexNewbuildingScraper,
|
||
_extract_coords,
|
||
_find_section_text,
|
||
_parse_corpus_count,
|
||
_parse_metro,
|
||
)
|
||
|
||
# ── Fixture HTML ──────────────────────────────────────────────────────────────
|
||
|
||
TATLIN_FIXTURE_HTML = """<!DOCTYPE html>
|
||
<html>
|
||
<head><title>ЖК «Татлин»</title></head>
|
||
<body>
|
||
<h1>ЖК «Татлин»</h1>
|
||
<div>Екатеринбург, ул. Черепанова / ул. Готвальда</div>
|
||
<a data-test="CARD_DEV_BADGE_DEVELOPER_LINK" href="/developer/prinzip">PRINZIP</a>
|
||
<div data-test="CardDevSites">
|
||
<a href="/zhk/1">ЖК Кислород</a>
|
||
<a href="/zhk/2">ЖК Нова</a>
|
||
</div>
|
||
<div>56.855312, 60.576668</div>
|
||
<div>Уральская 11 мин. Динамо 16 мин.</div>
|
||
<div>4.3 из 5 1505 оценок Смотреть все 353 отзыва</div>
|
||
<h2>О комплексе</h2>
|
||
<p>ЖК «Татлин» — три 35-этажные башни в Заречном микрорайоне Екатеринбурга.
|
||
Комплекс класса комфорт+. Введён в эксплуатацию в июне 2023 года.
|
||
Расположен на участке 1,5 га. Монолитный тип дома.</p>
|
||
<h2>Расположение</h2>
|
||
<p>В 5 минутах от центра</p>
|
||
</body>
|
||
</html>"""
|
||
|
||
# Minimal fixture — missing optional sections
|
||
MINIMAL_FIXTURE_HTML = """<!DOCTYPE html>
|
||
<html><body>
|
||
<h1>ЖК Минимальный</h1>
|
||
</body></html>"""
|
||
|
||
|
||
# ── Full happy-path test ──────────────────────────────────────────────────────
|
||
|
||
|
||
def test_parse_jk_full_fixture():
|
||
"""Happy path: all major fields extracted from Татлин-like HTML."""
|
||
scraper = YandexNewbuildingScraper()
|
||
info = scraper.parse(
|
||
TATLIN_FIXTURE_HTML,
|
||
jk_slug="tatlin",
|
||
jk_id="1592987",
|
||
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
|
||
)
|
||
|
||
assert isinstance(info, YandexNewbuildingInfo)
|
||
assert info.ext_id == "1592987"
|
||
assert info.ext_slug == "tatlin"
|
||
assert "Татлин" in (info.name or "")
|
||
|
||
# coords
|
||
assert info.lat == pytest.approx(56.855312, abs=1e-5)
|
||
assert info.lon == pytest.approx(60.576668, abs=1e-5)
|
||
|
||
# class + commission
|
||
assert info.house_class == "comfort_plus"
|
||
assert info.commission_year == 2023
|
||
assert info.commission_month == "июне"
|
||
|
||
# footprint
|
||
assert info.total_floors == 35
|
||
assert info.corpus_count == 3
|
||
assert info.total_area_ha == pytest.approx(1.5, abs=0.01)
|
||
|
||
# developer
|
||
assert info.developer_name == "PRINZIP"
|
||
assert info.developer_url is not None and "prinzip" in info.developer_url
|
||
assert "ЖК Кислород" in info.developer_other_jk
|
||
assert "ЖК Нова" in info.developer_other_jk
|
||
|
||
# reviews
|
||
assert info.rating == pytest.approx(4.3, abs=0.01)
|
||
assert info.ratings_count == 1505
|
||
assert info.text_reviews_count == 353
|
||
|
||
# description present
|
||
assert info.description is not None
|
||
assert len(info.description) > 10
|
||
|
||
# metro
|
||
assert len(info.metro_stations) >= 1
|
||
station_names = [s.name for s in info.metro_stations]
|
||
assert any("Уральская" in n for n in station_names)
|
||
|
||
# house type from "монолитный"
|
||
assert info.house_type == "monolith"
|
||
|
||
# raw_payload
|
||
assert info.raw_payload is not None
|
||
assert "body_len" in info.raw_payload
|
||
|
||
|
||
# ── Coord extraction tests ────────────────────────────────────────────────────
|
||
|
||
|
||
def test_extract_coords_within_ekb_range():
|
||
html = "<div>56.855312, 60.576668</div>"
|
||
lat, lon = _extract_coords(html)
|
||
assert lat == pytest.approx(56.855312, abs=1e-5)
|
||
assert lon == pytest.approx(60.576668, abs=1e-5)
|
||
|
||
|
||
def test_extract_coords_outside_range_returns_none():
|
||
# Moscow coords — outside EKB range
|
||
html = "<div>55.7558, 37.6176</div>"
|
||
_lat, lon = _extract_coords(html)
|
||
# 55.75 is in LAT_RANGE but 37.61 is NOT in LON_RANGE
|
||
assert lon is None
|
||
|
||
|
||
def test_extract_coords_no_coords_returns_none():
|
||
html = "<div>Текст без координат</div>"
|
||
lat, lon = _extract_coords(html)
|
||
assert lat is None
|
||
assert lon is None
|
||
|
||
|
||
def test_extract_coords_multiple_takes_first():
|
||
"""When multiple valid coords present, first is returned."""
|
||
html = "<div>56.855312, 60.576668</div><div>56.900000, 60.600000</div>"
|
||
lat, lon = _extract_coords(html)
|
||
assert lat == pytest.approx(56.855312, abs=1e-5)
|
||
assert lon == pytest.approx(60.576668, abs=1e-5)
|
||
|
||
|
||
# ── Corpus count tests ────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_parse_corpus_count_word_three():
|
||
text = "три 35-этажные башни в центре"
|
||
result = _parse_corpus_count(text)
|
||
assert result == 3
|
||
|
||
|
||
def test_parse_corpus_count_digit():
|
||
text = "5 35-этажных корпусов"
|
||
result = _parse_corpus_count(text)
|
||
assert result == 5
|
||
|
||
|
||
def test_parse_corpus_count_two():
|
||
text = "две 20-этажные башни"
|
||
result = _parse_corpus_count(text)
|
||
assert result == 2
|
||
|
||
|
||
def test_parse_corpus_count_none_when_no_match():
|
||
text = "Обычный текст без этажей"
|
||
result = _parse_corpus_count(text)
|
||
assert result is None
|
||
|
||
|
||
# ── House class test ──────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_parse_house_class_comfort_plus_from_description():
|
||
from app.services.scrapers.yandex_helpers import parse_house_class
|
||
|
||
text = "Жилой комплекс класса комфорт+ в центре города"
|
||
result = parse_house_class(text)
|
||
assert result == "comfort_plus"
|
||
|
||
|
||
def test_parse_house_class_business():
|
||
from app.services.scrapers.yandex_helpers import parse_house_class
|
||
|
||
text = "ЖК класса бизнес"
|
||
result = parse_house_class(text)
|
||
assert result == "business"
|
||
|
||
|
||
# ── Commission year/month test ────────────────────────────────────────────────
|
||
|
||
|
||
def test_parse_commission_year_month():
|
||
"""Full pipeline: parse() correctly extracts commission_year and commission_month."""
|
||
scraper = YandexNewbuildingScraper()
|
||
info = scraper.parse(
|
||
TATLIN_FIXTURE_HTML,
|
||
jk_slug="tatlin",
|
||
jk_id="1592987",
|
||
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
|
||
)
|
||
assert info.commission_year == 2023
|
||
assert info.commission_month == "июне"
|
||
|
||
|
||
def test_parse_commission_sdan_variant():
|
||
"""Regex supports 'сдан в эксплуатацию' phrasing as well."""
|
||
html = """<html><body>
|
||
<h1>ЖК Тест</h1>
|
||
<h2>О комплексе</h2>
|
||
<p>Сдан в эксплуатацию в марте 2025 года.</p>
|
||
</body></html>"""
|
||
scraper = YandexNewbuildingScraper()
|
||
info = scraper.parse(html, jk_slug="test", jk_id="999", source_url="http://x")
|
||
assert info.commission_year == 2025
|
||
assert info.commission_month == "марте"
|
||
|
||
|
||
# ── Rating + counts test ──────────────────────────────────────────────────────
|
||
|
||
|
||
def test_parse_rating_and_counts():
|
||
scraper = YandexNewbuildingScraper()
|
||
info = scraper.parse(
|
||
TATLIN_FIXTURE_HTML,
|
||
jk_slug="tatlin",
|
||
jk_id="1592987",
|
||
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
|
||
)
|
||
assert info.rating == pytest.approx(4.3, abs=0.01)
|
||
assert info.ratings_count == 1505
|
||
assert info.text_reviews_count == 353
|
||
|
||
|
||
def test_parse_rating_comma_separator():
|
||
"""Rating '4,3 из 5' with comma handled correctly."""
|
||
html = "<html><body><div>4,3 из 5 200 оценок</div></body></html>"
|
||
scraper = YandexNewbuildingScraper()
|
||
info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x")
|
||
assert info.rating == pytest.approx(4.3, abs=0.01)
|
||
assert info.ratings_count == 200
|
||
|
||
|
||
def test_parse_rating_no_space_before_iz():
|
||
"""Yandex drift (#974): '4.3из 5' / '4,5из\xa05' — без пробела перед «из», nbsp внутри.
|
||
|
||
Регрессия 2026-06-15: live-страницы рендерили рейтинг как «4.3из 5» (без
|
||
разделителя), из-за чего \\s+ перед «из» не матчил → rating=NULL при
|
||
заполненных ratings_count. \\s* должен извлечь рейтинг в обеих формах.
|
||
"""
|
||
scraper = YandexNewbuildingScraper()
|
||
info = scraper.parse(
|
||
"<html><body><div>Отзывы4.3из 5Всего 1508 оценок</div></body></html>",
|
||
jk_slug="x",
|
||
jk_id="1",
|
||
source_url="http://x",
|
||
)
|
||
assert info.rating == pytest.approx(4.3, abs=0.01)
|
||
assert info.ratings_count == 1508
|
||
# nbsp-вариант (как в реальном body_text)
|
||
info2 = scraper.parse(
|
||
"<html><body><div>4,5из\xa05Всего 850 оценок</div></body></html>",
|
||
jk_slug="y",
|
||
jk_id="2",
|
||
source_url="http://y",
|
||
)
|
||
assert info2.rating == pytest.approx(4.5, abs=0.01)
|
||
|
||
|
||
# ── Developer link test ───────────────────────────────────────────────────────
|
||
|
||
|
||
def test_developer_name_extracted_from_data_test_link():
|
||
scraper = YandexNewbuildingScraper()
|
||
info = scraper.parse(
|
||
TATLIN_FIXTURE_HTML,
|
||
jk_slug="tatlin",
|
||
jk_id="1592987",
|
||
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
|
||
)
|
||
assert info.developer_name == "PRINZIP"
|
||
assert info.developer_url == "https://realty.yandex.ru/developer/prinzip"
|
||
|
||
|
||
def test_developer_absolute_url_kept_as_is():
|
||
html = """<html><body>
|
||
<a data-test="CARD_DEV_BADGE_DEVELOPER_LINK"
|
||
href="https://realty.yandex.ru/developer/abc">АБС Групп</a>
|
||
</body></html>"""
|
||
scraper = YandexNewbuildingScraper()
|
||
info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x")
|
||
assert info.developer_name == "АБС Групп"
|
||
assert info.developer_url == "https://realty.yandex.ru/developer/abc"
|
||
|
||
|
||
# ── Metro stations test ───────────────────────────────────────────────────────
|
||
|
||
|
||
def test_metro_stations_parsed():
|
||
scraper = YandexNewbuildingScraper()
|
||
info = scraper.parse(
|
||
TATLIN_FIXTURE_HTML,
|
||
jk_slug="tatlin",
|
||
jk_id="1592987",
|
||
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
|
||
)
|
||
assert len(info.metro_stations) >= 1
|
||
uralskaya = next((s for s in info.metro_stations if "Уральская" in s.name), None)
|
||
assert uralskaya is not None
|
||
assert uralskaya.walk_min == 11
|
||
|
||
|
||
def test_parse_metro_direct():
|
||
text = "Уральская 11 мин. Динамо 16 мин."
|
||
stations = _parse_metro(text)
|
||
assert len(stations) == 2
|
||
assert stations[0].name == "Уральская"
|
||
assert stations[0].walk_min == 11
|
||
assert stations[1].name == "Динамо"
|
||
assert stations[1].walk_min == 16
|
||
|
||
|
||
def test_parse_metro_caps_at_five():
|
||
text = "Альфа 5 мин. Бета 6 мин. Гамма 7 мин. " "Дельта 8 мин. Эпсилон 9 мин. Зета 10 мин."
|
||
stations = _parse_metro(text)
|
||
assert len(stations) == 5
|
||
|
||
|
||
# ── No description section test ───────────────────────────────────────────────
|
||
|
||
|
||
def test_no_description_section_returns_none():
|
||
"""When no О комплексе / Расположение section, description is None."""
|
||
scraper = YandexNewbuildingScraper()
|
||
info = scraper.parse(
|
||
MINIMAL_FIXTURE_HTML,
|
||
jk_slug="minimal",
|
||
jk_id="42",
|
||
source_url="http://example.com/",
|
||
)
|
||
assert info.description is None
|
||
assert info.house_class is None
|
||
assert info.commission_year is None
|
||
assert info.total_floors is None
|
||
|
||
|
||
# ── fetch_around raises NotImplementedError ───────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_raises_not_implemented():
|
||
scraper = YandexNewbuildingScraper()
|
||
with pytest.raises(NotImplementedError, match="JK-slug-based"):
|
||
await scraper.fetch_around(56.855, 60.576)
|
||
|
||
|
||
# ── _find_section_text helper ─────────────────────────────────────────────────
|
||
|
||
|
||
def test_find_section_text_returns_content_after_heading():
|
||
from selectolax.parser import HTMLParser
|
||
|
||
html = """<html><body>
|
||
<h2>О комплексе</h2>
|
||
<p>Описание комплекса здесь.</p>
|
||
<p>Ещё абзац.</p>
|
||
<h2>Другой раздел</h2>
|
||
<p>Не должен попасть.</p>
|
||
</body></html>"""
|
||
tree = HTMLParser(html)
|
||
result = _find_section_text(tree, "О комплексе")
|
||
assert result is not None
|
||
assert "Описание комплекса" in result
|
||
assert "Не должен" not in result
|
||
|
||
|
||
def test_find_section_text_returns_none_when_missing():
|
||
from selectolax.parser import HTMLParser
|
||
|
||
html = "<html><body><p>Текст</p></body></html>"
|
||
tree = HTMLParser(html)
|
||
result = _find_section_text(tree, "О комплексе")
|
||
assert result is None
|
||
|
||
|
||
# ── Developer other JK dedup ──────────────────────────────────────────────────
|
||
|
||
|
||
def test_developer_other_jk_dedup():
|
||
"""Duplicate JK names are not added twice."""
|
||
html = """<html><body>
|
||
<div data-test="CardDevSites">
|
||
<a href="/1">ЖК Один</a>
|
||
<a href="/2">ЖК Один</a>
|
||
<a href="/3">ЖК Два</a>
|
||
</div>
|
||
</body></html>"""
|
||
scraper = YandexNewbuildingScraper()
|
||
info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x")
|
||
assert info.developer_other_jk.count("ЖК Один") == 1
|
||
assert "ЖК Два" in info.developer_other_jk
|
||
|
||
|
||
def test_developer_other_jk_capped_at_10():
|
||
"""Only first 10 other JK entries are kept."""
|
||
links = "\n".join(f'<a href="/{i}">ЖК {i}</a>' for i in range(15))
|
||
html = f"""<html><body>
|
||
<div data-test="CardDevSites">{links}</div>
|
||
</body></html>"""
|
||
scraper = YandexNewbuildingScraper()
|
||
info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x")
|
||
assert len(info.developer_other_jk) == 10
|
||
|
||
|
||
# ── model defaults ────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_yandex_newbuilding_info_defaults():
|
||
"""Model initialises with safe defaults for all optional fields."""
|
||
info = YandexNewbuildingInfo(ext_id="1", ext_slug="x", source_url="http://x")
|
||
assert info.name is None
|
||
assert info.lat is None
|
||
assert info.lon is None
|
||
assert info.house_class is None
|
||
assert info.developer_other_jk == []
|
||
assert info.metro_stations == []
|
||
assert info.raw_payload is None
|
||
|
||
|
||
def test_jk_metro_station_model():
|
||
station = JKMetroStation(name="Уральская", walk_min=11)
|
||
assert station.name == "Уральская"
|
||
assert station.walk_min == 11
|
||
|
||
station_no_time = JKMetroStation(name="Геологическая")
|
||
assert station_no_time.walk_min is None
|