"""Unit tests for YandexDetailScraper — Product JSON-LD + DOM detail parser.
All tests run against hand-crafted HTML fixtures. No live network access.
Легаси `app.services.scrapers.yandex_detail` удалён (#2277 финальный шаг
scraper_kit-миграции, 0 runtime importers) — тесты переведены на kit-эквивалент
`scraper_kit.providers.yandex.detail` без изменения покрытия.
"""
from __future__ import annotations
import json
import pytest
from scraper_kit.providers.yandex.detail import (
DetailEnrichment,
MetroStation,
YandexDetailScraper,
_extract_relative_date,
_find_section_text,
_parse_metro_stations,
_parse_title,
)
# ---------------------------------------------------------------------------
# Helpers to build fixture HTML
# ---------------------------------------------------------------------------
_BASE_OFFER_URL = "https://realty.yandex.ru/offer/7812345000001/"
_PRODUCT_LD = {
"@type": "Product",
"name": "3-комнатная квартира 85,5 м² на 12 этаж из 24",
"image": [
"https://avatars.mds.yandex.net/get-realty/photo1/main",
"https://avatars.mds.yandex.net/get-realty/photo2/main",
"https://avatars.mds.yandex.net/get-realty/photo3/main",
],
"offers": {
"@type": "Offer",
"price": 9850000,
"priceCurrency": "RUB",
},
}
def _make_html(
*,
product_ld: dict | None = _PRODUCT_LD,
h1: str = "3-комнатная квартира 85,5 м², 12 этаж из 24",
summary_text: str = (
"Россия, Свердловская область, Екатеринбург, улица Малышева, д. 5"
" • 115 233 ₽ за м²"
" • свободная продажа"
" • 342 просмотра"
" • опубликовано 10 апреля 2026"
),
author_block: str = "",
description_text: str = "Продаётся просторная кирпичная квартира 1995 года постройки.",
location_text: str = "Уральская 11 мин. Динамо 16 мин.",
nb_href: str | None = "/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
extra_head: str = "",
) -> str:
ld_block = ""
if product_ld is not None:
ld_block = f''
nb_link = ""
if nb_href:
nb_link = f'ЖК Татлин'
return f"""
{extra_head}{ld_block}
{h1}
{summary_text}
{author_block}
Описание
{description_text}
Расположение
{location_text}
{nb_link}
"""
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
FULL_HTML = _make_html()
SCRAPER = YandexDetailScraper()
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
class TestFullDetailFixture:
"""Happy path — all fields populated."""
def test_parse_full_detail_fixture(self) -> None:
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
assert isinstance(result, DetailEnrichment)
assert result.offer_id == "7812345000001"
assert result.source_url == _BASE_OFFER_URL
def test_price_from_json_ld(self) -> None:
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.price_rub == 9_850_000
def test_title_and_room_parsing(self) -> None:
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.rooms == 3
assert result.area_m2 == pytest.approx(85.5)
assert result.floor == 12
assert result.total_floors == 24
def test_views_and_publish_date(self) -> None:
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
assert result is not None
from datetime import date
assert result.views_total == 342
assert result.publish_date == date(2026, 4, 10)
def test_price_per_m2_from_summary(self) -> None:
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.price_per_m2 == 115_233
def test_address_extraction(self) -> None:
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.address is not None
assert "Малышева" in result.address
def test_description_section(self) -> None:
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.description is not None
assert "кирпичная" in result.description
def test_nlp_house_type_from_description(self) -> None:
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.house_type_nlp == "brick"
def test_year_built_hint_nlp(self) -> None:
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.year_built_hint == 1995
def test_raw_payload_present(self) -> None:
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.raw_payload is not None
assert "summary_text" in result.raw_payload
assert "photo_count" in result.raw_payload
class TestNoProductLD:
"""Missing JSON-LD — scraper falls back to summary / DOM only."""
def test_parse_no_product_ld_uses_summary_fallbacks(self) -> None:
html = _make_html(product_ld=None)
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
assert result is not None
# price_rub is None when no JSON-LD and no ₽ total in summary
assert result.price_rub is None
# price_per_m2 still extracted from "₽ за м²" in summary
assert result.price_per_m2 == 115_233
# photos array empty when no JSON-LD image[]
assert result.photo_urls == []
class TestStudio:
def test_parse_studio_rooms_zero(self) -> None:
html = _make_html(
product_ld=None,
h1="Студия 28 м², 2 этаж из 9",
summary_text="альтернативная • 50 просмотров",
)
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.rooms == 0
assert result.area_m2 == pytest.approx(28.0)
assert result.floor == 2
assert result.total_floors == 9
class TestNoAgencySection:
def test_parse_no_agency_section(self) -> None:
html = _make_html(author_block="") # no OfferCardAuthorInfo
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.agency_name is None
assert result.agency_founded_year is None
assert result.agency_objects_count is None
assert result.seller_name is None
class TestAgencyBlock:
def test_agency_fields_parsed(self) -> None:
# Note: selectolax .text(strip=True) concatenates text nodes without spaces.
# "Год основания 1998" + "150 объектов" becomes "Год основания 1998150 объектов".
# RE_AGENCY_OBJECTS = r"(\d+)\s+объект" would then match "1998150 объект".
# To avoid this ambiguity the fixture uses а non-digit separator between spans.
author_html = """
АН Городской риелтор
Год основания 1998. Продали 150 объектов.
"""
summary = "Иван Петров АН Городской риелтор • 50 просмотров • опубликовано 3 марта 2025"
html = _make_html(author_block=author_html, summary_text=summary)
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.agency_name == "АН Городской риелтор"
assert result.agency_founded_year == 1998
assert result.agency_objects_count == 150
class TestSellerName:
def test_seller_name_before_agency(self) -> None:
author_html = """
АН Капитал
"""
# seller name appears right before agency name in summary
summary = "Мария Кузнецова АН Капитал • 10 просмотров"
html = _make_html(author_block=author_html, summary_text=summary)
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.seller_name is not None
assert "Кузнецова" in result.seller_name
class TestNewbuildingLink:
def test_parse_newbuilding_link_extracted(self) -> None:
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.newbuilding_id == "1592987"
assert result.newbuilding_url is not None
assert "tatlin" in result.newbuilding_url
def test_no_newbuilding_link(self) -> None:
html = _make_html(nb_href=None)
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.newbuilding_id is None
assert result.newbuilding_url is None
class TestMetroStations:
def test_parse_metro_stations_multiple(self) -> None:
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
assert result is not None
assert len(result.metro_stations) == 2
names = [s.name for s in result.metro_stations]
assert "Уральская" in names
# walk_min for Уральская = 11
ural = next(s for s in result.metro_stations if s.name == "Уральская")
assert ural.walk_min == 11
def test_metro_stations_empty_when_no_location(self) -> None:
html = _make_html(location_text="")
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.metro_stations == []
def test_parse_metro_stations_helper_directly(self) -> None:
text = "Проспект Космонавтов 7 мин. Уралмаш 14 мин. Эльмаш 20 мин."
stations = _parse_metro_stations(text)
assert len(stations) == 3
assert stations[0] == MetroStation(name="Проспект Космонавтов", walk_min=7)
assert stations[1].walk_min == 14
class TestRelativeDate:
def test_parse_relative_date_yesterday(self) -> None:
html = _make_html(summary_text="Россия, ЕКБ, ул. Тест • 100 просмотров • вчера")
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.publish_date_relative == "вчера"
def test_relative_date_hours(self) -> None:
result = _extract_relative_date("опубликовано 6 часов назад")
assert result == "6 часов назад"
def test_relative_date_days(self) -> None:
result = _extract_relative_date("размещено 3 дня назад и ещё текст")
assert result == "3 дня назад"
def test_relative_date_none(self) -> None:
result = _extract_relative_date("опубликовано 10 апреля 2026")
assert result is None
class TestPublishDate:
def test_publish_date_ru_format(self) -> None:
html = _make_html(summary_text="Россия, ЕКБ • 200 просмотров • опубликовано 5 января 2026")
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
assert result is not None
from datetime import date
assert result.publish_date == date(2026, 1, 5)
class TestPhotos:
def test_photos_from_json_ld_image_array(self) -> None:
ld = dict(_PRODUCT_LD)
ld["image"] = [
"https://avatars.mds.yandex.net/get-realty/img1",
"https://avatars.mds.yandex.net/get-realty/img2",
"https://avatars.mds.yandex.net/get-realty/img3",
"https://avatars.mds.yandex.net/get-realty/img4",
"https://avatars.mds.yandex.net/get-realty/img5",
"https://avatars.mds.yandex.net/get-realty/img6",
"https://avatars.mds.yandex.net/get-realty/img7",
"https://avatars.mds.yandex.net/get-realty/img8",
]
html = _make_html(product_ld=ld)
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
assert result is not None
assert len(result.photo_urls) == 8
assert all(u.startswith("https://") for u in result.photo_urls)
def test_photo_single_string_wrapped(self) -> None:
ld = dict(_PRODUCT_LD)
ld["image"] = "https://avatars.mds.yandex.net/get-realty/single"
html = _make_html(product_ld=ld)
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.photo_urls == ["https://avatars.mds.yandex.net/get-realty/single"]
class TestInvalidUrl:
def test_invalid_offer_url_returns_none(self) -> None:
result = SCRAPER.parse(FULL_HTML, offer_url="https://realty.yandex.ru/ekaterinburg/")
assert result is None
def test_valid_url_with_trailing_slash(self) -> None:
result = SCRAPER.parse(FULL_HTML, offer_url="https://realty.yandex.ru/offer/999/")
assert result is not None
assert result.offer_id == "999"
class TestHelpers:
def test_parse_title_full(self) -> None:
rooms, area, floor, total = _parse_title("2-комнатная квартира 55,3 м², 7 этаж из 18")
assert rooms == 2
assert area == pytest.approx(55.3)
assert floor == 7
assert total == 18
def test_parse_title_studio(self) -> None:
rooms, area, _floor, _total = _parse_title("Студия 28 м², 2 этаж из 9")
assert rooms == 0
assert area == pytest.approx(28.0)
def test_parse_title_missing_floor(self) -> None:
rooms, area, floor, total = _parse_title("1-комнатная квартира 36 м²")
assert rooms == 1
assert area == pytest.approx(36.0)
assert floor is None
assert total is None
def test_find_section_text_returns_none_when_absent(self) -> None:
from selectolax.parser import HTMLParser
tree = HTMLParser("Другой раздел
текст
")
assert _find_section_text(tree, "Описание") is None
def test_metro_walk_min_from_nlp(self) -> None:
"""metro_walk_min extracted from description via RE_METRO_WALK."""
html = _make_html(
description_text="Квартира в 7 минут неспешной прогулки от метро.",
location_text="",
)
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.metro_walk_min == 7
# ---------------------------------------------------------------------------
# #2669 — просмотры с разделителем тысяч
# ---------------------------------------------------------------------------
# Форма взята дословно из сохранённого ответа Яндекса
# tests/fixtures/yandex_offer_3402418396407468801.html (сверено 2026-08-07):
# 16 июня, 8 просмотров35,5 м², ...
# 3 850 000 ₽108 451 ₽ за м²...
# — тот же блок, из которого читаются просмотры, отдаёт числа с NBSP-разделителем.
# Прежняя RE_VIEWS `(\d+)\s+просмотр` обрывалась на разделителе и писала в БД
# ПОСЛЕДНИЕ ТРИ ЦИФРЫ (1 234 → 234): не пусто, а тихо неверно. Косвенное
# подтверждение с прода на 2026-08-07: из 1620 непустых views_total_yandex
# ровно НОЛЬ значений ≥ 1000, при 2512 таких у avito.
@pytest.mark.parametrize(
("views_fragment", "expected"),
[
("8\xa0просмотров", 8), # дословно из сохранённого ответа
("1\xa0234 просмотра", 1234), # NBSP-разделитель — до правки давало 234
("1 234 просмотра", 1234), # обычный пробел
("12\xa0345 просмотров", 12345),
("342 просмотра", 342), # без разделителя — прежнее поведение цело
],
)
def test_views_total_survives_thousands_separator(views_fragment: str, expected: int) -> None:
html = _make_html(summary_text=f"Екатеринбург, улица Малышева, д. 5 • {views_fragment}")
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.views_total == expected, "просмотры Яндекса снова режутся по разделителю тысяч"
def test_views_total_absent_stays_none() -> None:
"""Нет блока просмотров → None, а не 0: «не пришло» ≠ «ноль просмотров»."""
html = _make_html(summary_text="Екатеринбург, улица Малышева, д. 5 • нет данных")
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
assert result is not None
assert result.views_total is None