a9b87aa311
All checks were successful
CI Trade-In / changes (pull_request) Successful in 8s
CI / changes (pull_request) Successful in 8s
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
CI Trade-In / backend-tests (pull_request) Successful in 1m36s
Удалены (0 runtime importers в app/, scripts/, packages/; проверено grep'ом, включая intra-legacy импорты в оставшихся 21 legacy-модуле scrapers/): - backend/app/services/scrapers/avito_imv.py - backend/app/services/scrapers/domclick_detail.py - backend/app/services/scrapers/ekb_geoportal_client.py - backend/app/services/scrapers/yandex_detail.py - backend/app/services/scrapers/yandex_valuation.py Тесты — удалены (pure legacy-vs-kit parity, kit-only coverage уже есть в другом месте): - tests/scrapers/test_ekb_geoportal_client_kit_parity.py - tests/scrapers/test_domclick_detail_kit_parity.py Тесты — конвертированы в kit-only (импорт легаси заменён на scraper_kit.*, покрытие сохранено без потерь): - tests/scrapers/test_avito_imv_kit_parity.py (parity-тесты убраны, config-footgun regression на kit-стороне оставлен) - tests/scrapers/test_avito_imv_browser_transport.py - tests/scrapers/test_yandex_valuation_kit_migration.py (parity убрана, footgun- регрессия mandatory config/delay_provider оставлена) - tests/scrapers/test_domclick_detail.py (+ exceptions переведены на scraper_kit.domclick_exceptions, т.к. kit parse_detail_html поднимает их) - tests/test_avito_imv_parse.py - tests/test_ekb_geoportal_ingest.py (client-часть; ingest/geocoder-часть не трогали — не зависят от удалённого модуля) - tests/test_yandex_detail.py - tests/test_yandex_detail_structural.py - tests/test_yandex_valuation.py (YandexValuationScraper() → config=_KIT_CONFIG, kit конструктор требует обязательный config) - tests/test_yandex_valuation_save.py - tests/test_extval_house_id_write_path.py (только yandex_valuation-часть; cian_valuation остался нетронутым — живой легаси-модуль) - tests/test_yandex_history_area_filter.py Тесты — частично отредактированы (убраны только части про удалённые модули, живые легаси-модули/их parity не тронуты): - tests/scrapers/test_admin_domclick_ingest_kit_parity.py (base.py/ScrapedLot parity остался) - tests/scrapers/test_admin_yandex_kit_parity.py (yandex_realty parity остался) - tests/scrapers/test_admin_avito_kit_parity.py (avito.py/avito_houses.py parity остался; _parse_price parity убран) - tests/scrapers/test_avito_unix_date_tz_consistency.py (IMV-ветка переведена на kit avito.imv, легаси avito/avito_houses/avito_shared/avito_detail не тронуты) - tests/tasks/test_yandex_detail_backfill.py (subject — kit-задача, уже вызывает scraper_kit.providers.yandex.detail; save_detail_enrichment coverage-тесты внизу файла переведены на kit) - tests/test_scraper_kit_yandex_golden_parity.py (detail/valuation parity убраны; serp/newbuilding/house_type_normalizer/build_url для yandex_realty/ yandex_newbuilding не тронуты) - tests/test_scraper_proxy.py (YandexValuation proxy-тесты переведены на kit config= DI; test_avito_imv_own_session_receives_proxies удалён — дублирует test_avito_imv_kit_parity.py footgun-тесты) - tests/test_yandex_scrapers_delay_wiring.py (yandex_detail/yandex_valuation → kit delay_provider=; yandex_realty/yandex_newbuilding не тронуты) - tests/test_scraper_kit_group_c_backfill_kit_parity.py (docstring уточнение, без функциональных изменений) Полный backend-suite зелёный (3253 passed, 6 skipped) кроме известного pre-existing flake test_search_api.py::test_search_cache_hit (#2208, не регрессия этого PR). ruff check + ruff format — чисто на всех изменённых файлах.
387 lines
15 KiB
Python
387 lines
15 KiB
Python
"""Unit tests for YandexDetailScraper — Product JSON-LD + DOM detail parser.
|
||
|
||
All tests run against hand-crafted HTML fixtures. No live network access.
|
||
|
||
Легаси `app.services.scrapers.yandex_detail` удалён (#2277 финальный шаг
|
||
scraper_kit-миграции, 0 runtime importers) — тесты переведены на kit-эквивалент
|
||
`scraper_kit.providers.yandex.detail` без изменения покрытия.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
|
||
import pytest
|
||
from scraper_kit.providers.yandex.detail import (
|
||
DetailEnrichment,
|
||
MetroStation,
|
||
YandexDetailScraper,
|
||
_extract_relative_date,
|
||
_find_section_text,
|
||
_parse_metro_stations,
|
||
_parse_title,
|
||
)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Helpers to build fixture HTML
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_BASE_OFFER_URL = "https://realty.yandex.ru/offer/7812345000001/"
|
||
|
||
_PRODUCT_LD = {
|
||
"@type": "Product",
|
||
"name": "3-комнатная квартира 85,5 м² на 12 этаж из 24",
|
||
"image": [
|
||
"https://avatars.mds.yandex.net/get-realty/photo1/main",
|
||
"https://avatars.mds.yandex.net/get-realty/photo2/main",
|
||
"https://avatars.mds.yandex.net/get-realty/photo3/main",
|
||
],
|
||
"offers": {
|
||
"@type": "Offer",
|
||
"price": 9850000,
|
||
"priceCurrency": "RUB",
|
||
},
|
||
}
|
||
|
||
|
||
def _make_html(
|
||
*,
|
||
product_ld: dict | None = _PRODUCT_LD,
|
||
h1: str = "3-комнатная квартира 85,5 м², 12 этаж из 24",
|
||
summary_text: str = (
|
||
"Россия, Свердловская область, Екатеринбург, улица Малышева, д. 5"
|
||
" • 115 233 ₽ за м²"
|
||
" • свободная продажа"
|
||
" • 342 просмотра"
|
||
" • опубликовано 10 апреля 2026"
|
||
),
|
||
author_block: str = "",
|
||
description_text: str = "Продаётся просторная кирпичная квартира 1995 года постройки.",
|
||
location_text: str = "Уральская 11 мин. Динамо 16 мин.",
|
||
nb_href: str | None = "/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
|
||
extra_head: str = "",
|
||
) -> str:
|
||
ld_block = ""
|
||
if product_ld is not None:
|
||
ld_block = f'<script type="application/ld+json">{json.dumps(product_ld)}</script>'
|
||
|
||
nb_link = ""
|
||
if nb_href:
|
||
nb_link = f'<a href="{nb_href}">ЖК Татлин</a>'
|
||
|
||
return f"""<!DOCTYPE html>
|
||
<html>
|
||
<head>{extra_head}{ld_block}</head>
|
||
<body>
|
||
<h1>{h1}</h1>
|
||
<div data-test="OfferCardSummary">{summary_text}</div>
|
||
{author_block}
|
||
<h2>Описание</h2>
|
||
<p>{description_text}</p>
|
||
<h2>Расположение</h2>
|
||
<p>{location_text}</p>
|
||
{nb_link}
|
||
</body>
|
||
</html>"""
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Fixtures
|
||
# ---------------------------------------------------------------------------
|
||
|
||
FULL_HTML = _make_html()
|
||
|
||
SCRAPER = YandexDetailScraper()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Tests
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class TestFullDetailFixture:
|
||
"""Happy path — all fields populated."""
|
||
|
||
def test_parse_full_detail_fixture(self) -> None:
|
||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||
assert isinstance(result, DetailEnrichment)
|
||
assert result.offer_id == "7812345000001"
|
||
assert result.source_url == _BASE_OFFER_URL
|
||
|
||
def test_price_from_json_ld(self) -> None:
|
||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert result.price_rub == 9_850_000
|
||
|
||
def test_title_and_room_parsing(self) -> None:
|
||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert result.rooms == 3
|
||
assert result.area_m2 == pytest.approx(85.5)
|
||
assert result.floor == 12
|
||
assert result.total_floors == 24
|
||
|
||
def test_views_and_publish_date(self) -> None:
|
||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
from datetime import date
|
||
|
||
assert result.views_total == 342
|
||
assert result.publish_date == date(2026, 4, 10)
|
||
|
||
def test_price_per_m2_from_summary(self) -> None:
|
||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert result.price_per_m2 == 115_233
|
||
|
||
def test_address_extraction(self) -> None:
|
||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert result.address is not None
|
||
assert "Малышева" in result.address
|
||
|
||
def test_description_section(self) -> None:
|
||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert result.description is not None
|
||
assert "кирпичная" in result.description
|
||
|
||
def test_nlp_house_type_from_description(self) -> None:
|
||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert result.house_type_nlp == "brick"
|
||
|
||
def test_year_built_hint_nlp(self) -> None:
|
||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert result.year_built_hint == 1995
|
||
|
||
def test_raw_payload_present(self) -> None:
|
||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert result.raw_payload is not None
|
||
assert "summary_text" in result.raw_payload
|
||
assert "photo_count" in result.raw_payload
|
||
|
||
|
||
class TestNoProductLD:
|
||
"""Missing JSON-LD — scraper falls back to summary / DOM only."""
|
||
|
||
def test_parse_no_product_ld_uses_summary_fallbacks(self) -> None:
|
||
html = _make_html(product_ld=None)
|
||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
# price_rub is None when no JSON-LD and no ₽ total in summary
|
||
assert result.price_rub is None
|
||
# price_per_m2 still extracted from "₽ за м²" in summary
|
||
assert result.price_per_m2 == 115_233
|
||
# photos array empty when no JSON-LD image[]
|
||
assert result.photo_urls == []
|
||
|
||
|
||
class TestStudio:
|
||
def test_parse_studio_rooms_zero(self) -> None:
|
||
html = _make_html(
|
||
product_ld=None,
|
||
h1="Студия 28 м², 2 этаж из 9",
|
||
summary_text="альтернативная • 50 просмотров",
|
||
)
|
||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert result.rooms == 0
|
||
assert result.area_m2 == pytest.approx(28.0)
|
||
assert result.floor == 2
|
||
assert result.total_floors == 9
|
||
|
||
|
||
class TestNoAgencySection:
|
||
def test_parse_no_agency_section(self) -> None:
|
||
html = _make_html(author_block="") # no OfferCardAuthorInfo
|
||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert result.agency_name is None
|
||
assert result.agency_founded_year is None
|
||
assert result.agency_objects_count is None
|
||
assert result.seller_name is None
|
||
|
||
|
||
class TestAgencyBlock:
|
||
def test_agency_fields_parsed(self) -> None:
|
||
# Note: selectolax .text(strip=True) concatenates text nodes without spaces.
|
||
# "Год основания 1998" + "150 объектов" becomes "Год основания 1998150 объектов".
|
||
# RE_AGENCY_OBJECTS = r"(\d+)\s+объект" would then match "1998150 объект".
|
||
# To avoid this ambiguity the fixture uses а non-digit separator between spans.
|
||
author_html = """
|
||
<div data-test="OfferCardAuthorInfo">
|
||
<h2>АН Городской риелтор</h2>
|
||
<p>Год основания 1998. Продали 150 объектов.</p>
|
||
</div>"""
|
||
summary = "Иван Петров АН Городской риелтор • 50 просмотров • опубликовано 3 марта 2025"
|
||
html = _make_html(author_block=author_html, summary_text=summary)
|
||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert result.agency_name == "АН Городской риелтор"
|
||
assert result.agency_founded_year == 1998
|
||
assert result.agency_objects_count == 150
|
||
|
||
|
||
class TestSellerName:
|
||
def test_seller_name_before_agency(self) -> None:
|
||
author_html = """
|
||
<div data-test="OfferCardAuthorInfo">
|
||
<h2>АН Капитал</h2>
|
||
</div>"""
|
||
# seller name appears right before agency name in summary
|
||
summary = "Мария Кузнецова АН Капитал • 10 просмотров"
|
||
html = _make_html(author_block=author_html, summary_text=summary)
|
||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert result.seller_name is not None
|
||
assert "Кузнецова" in result.seller_name
|
||
|
||
|
||
class TestNewbuildingLink:
|
||
def test_parse_newbuilding_link_extracted(self) -> None:
|
||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert result.newbuilding_id == "1592987"
|
||
assert result.newbuilding_url is not None
|
||
assert "tatlin" in result.newbuilding_url
|
||
|
||
def test_no_newbuilding_link(self) -> None:
|
||
html = _make_html(nb_href=None)
|
||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert result.newbuilding_id is None
|
||
assert result.newbuilding_url is None
|
||
|
||
|
||
class TestMetroStations:
|
||
def test_parse_metro_stations_multiple(self) -> None:
|
||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert len(result.metro_stations) == 2
|
||
names = [s.name for s in result.metro_stations]
|
||
assert "Уральская" in names
|
||
# walk_min for Уральская = 11
|
||
ural = next(s for s in result.metro_stations if s.name == "Уральская")
|
||
assert ural.walk_min == 11
|
||
|
||
def test_metro_stations_empty_when_no_location(self) -> None:
|
||
html = _make_html(location_text="")
|
||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert result.metro_stations == []
|
||
|
||
def test_parse_metro_stations_helper_directly(self) -> None:
|
||
text = "Проспект Космонавтов 7 мин. Уралмаш 14 мин. Эльмаш 20 мин."
|
||
stations = _parse_metro_stations(text)
|
||
assert len(stations) == 3
|
||
assert stations[0] == MetroStation(name="Проспект Космонавтов", walk_min=7)
|
||
assert stations[1].walk_min == 14
|
||
|
||
|
||
class TestRelativeDate:
|
||
def test_parse_relative_date_yesterday(self) -> None:
|
||
html = _make_html(summary_text="Россия, ЕКБ, ул. Тест • 100 просмотров • вчера")
|
||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert result.publish_date_relative == "вчера"
|
||
|
||
def test_relative_date_hours(self) -> None:
|
||
result = _extract_relative_date("опубликовано 6 часов назад")
|
||
assert result == "6 часов назад"
|
||
|
||
def test_relative_date_days(self) -> None:
|
||
result = _extract_relative_date("размещено 3 дня назад и ещё текст")
|
||
assert result == "3 дня назад"
|
||
|
||
def test_relative_date_none(self) -> None:
|
||
result = _extract_relative_date("опубликовано 10 апреля 2026")
|
||
assert result is None
|
||
|
||
|
||
class TestPublishDate:
|
||
def test_publish_date_ru_format(self) -> None:
|
||
html = _make_html(summary_text="Россия, ЕКБ • 200 просмотров • опубликовано 5 января 2026")
|
||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
from datetime import date
|
||
|
||
assert result.publish_date == date(2026, 1, 5)
|
||
|
||
|
||
class TestPhotos:
|
||
def test_photos_from_json_ld_image_array(self) -> None:
|
||
ld = dict(_PRODUCT_LD)
|
||
ld["image"] = [
|
||
"https://avatars.mds.yandex.net/get-realty/img1",
|
||
"https://avatars.mds.yandex.net/get-realty/img2",
|
||
"https://avatars.mds.yandex.net/get-realty/img3",
|
||
"https://avatars.mds.yandex.net/get-realty/img4",
|
||
"https://avatars.mds.yandex.net/get-realty/img5",
|
||
"https://avatars.mds.yandex.net/get-realty/img6",
|
||
"https://avatars.mds.yandex.net/get-realty/img7",
|
||
"https://avatars.mds.yandex.net/get-realty/img8",
|
||
]
|
||
html = _make_html(product_ld=ld)
|
||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert len(result.photo_urls) == 8
|
||
assert all(u.startswith("https://") for u in result.photo_urls)
|
||
|
||
def test_photo_single_string_wrapped(self) -> None:
|
||
ld = dict(_PRODUCT_LD)
|
||
ld["image"] = "https://avatars.mds.yandex.net/get-realty/single"
|
||
html = _make_html(product_ld=ld)
|
||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert result.photo_urls == ["https://avatars.mds.yandex.net/get-realty/single"]
|
||
|
||
|
||
class TestInvalidUrl:
|
||
def test_invalid_offer_url_returns_none(self) -> None:
|
||
result = SCRAPER.parse(FULL_HTML, offer_url="https://realty.yandex.ru/ekaterinburg/")
|
||
assert result is None
|
||
|
||
def test_valid_url_with_trailing_slash(self) -> None:
|
||
result = SCRAPER.parse(FULL_HTML, offer_url="https://realty.yandex.ru/offer/999/")
|
||
assert result is not None
|
||
assert result.offer_id == "999"
|
||
|
||
|
||
class TestHelpers:
|
||
def test_parse_title_full(self) -> None:
|
||
rooms, area, floor, total = _parse_title("2-комнатная квартира 55,3 м², 7 этаж из 18")
|
||
assert rooms == 2
|
||
assert area == pytest.approx(55.3)
|
||
assert floor == 7
|
||
assert total == 18
|
||
|
||
def test_parse_title_studio(self) -> None:
|
||
rooms, area, _floor, _total = _parse_title("Студия 28 м², 2 этаж из 9")
|
||
assert rooms == 0
|
||
assert area == pytest.approx(28.0)
|
||
|
||
def test_parse_title_missing_floor(self) -> None:
|
||
rooms, area, floor, total = _parse_title("1-комнатная квартира 36 м²")
|
||
assert rooms == 1
|
||
assert area == pytest.approx(36.0)
|
||
assert floor is None
|
||
assert total is None
|
||
|
||
def test_find_section_text_returns_none_when_absent(self) -> None:
|
||
from selectolax.parser import HTMLParser
|
||
|
||
tree = HTMLParser("<html><body><h2>Другой раздел</h2><p>текст</p></body></html>")
|
||
assert _find_section_text(tree, "Описание") is None
|
||
|
||
def test_metro_walk_min_from_nlp(self) -> None:
|
||
"""metro_walk_min extracted from description via RE_METRO_WALK."""
|
||
html = _make_html(
|
||
description_text="Квартира в 7 минут неспешной прогулки от метро.",
|
||
location_text="",
|
||
)
|
||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||
assert result is not None
|
||
assert result.metro_walk_min == 7
|