242 lines
8.7 KiB
Python
242 lines
8.7 KiB
Python
"""Unit tests for YandexValuationScraper — anonymous house-history scraper.
|
||
|
||
Fixture HTML simulates the Yandex valuation page body text containing:
|
||
- House meta block (year, floors, type, ceiling, lift, total objects, panorama)
|
||
- 2-3 historical offer entries with full structure
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from datetime import date
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.yandex_valuation import (
|
||
YandexValuationResult,
|
||
YandexValuationScraper,
|
||
)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Fixture helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_HOUSE_META_BLOCK = (
|
||
"12 объектов Дом 1981 года 9 этажей Панельное здание 2,50 м потолки Лифт"
|
||
)
|
||
|
||
_ITEM_1 = (
|
||
"2-комнатная квартира 45,3 м² 4 этаж "
|
||
"Опубликовано 15.03.2023 "
|
||
"Начальная цена 3 200 000 ₽ 117 000 ₽ за м² "
|
||
"Последняя цена 3 100 000 ₽ 114 000 ₽ за м² "
|
||
"Длительность экспозиции 42 дня В продаже"
|
||
)
|
||
|
||
_ITEM_2 = (
|
||
"Студия 25,0 м² 1 этаж "
|
||
"Опубликовано 20.11.2022 "
|
||
"Начальная цена 2 100 000 ₽ 84 000 ₽ за м² "
|
||
"Последняя цена 1 950 000 ₽ 78 000 ₽ за м² "
|
||
"Длительность экспозиции 90 дней Снято"
|
||
)
|
||
|
||
_ITEM_3 = (
|
||
"1-комнатная квартира 32,5 м² 7 этаж "
|
||
"Опубликовано 01.06.2024 "
|
||
"Начальная цена 2 800 000 ₽ "
|
||
"Длительность экспозиции 15 дней В продаже"
|
||
)
|
||
|
||
|
||
def _make_full_html(body_content: str) -> str:
|
||
return f"<html><body>{body_content}</body></html>"
|
||
|
||
|
||
FULL_FIXTURE_HTML = _make_full_html(
|
||
f"{_HOUSE_META_BLOCK}\n{_ITEM_1}\n{_ITEM_2}\n{_ITEM_3}"
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# House meta tests
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_house_meta_full():
|
||
scraper = YandexValuationScraper()
|
||
meta = scraper._parse_house_meta(_HOUSE_META_BLOCK)
|
||
assert meta.year_built == 1981
|
||
assert meta.total_floors == 9
|
||
assert meta.house_type == "panel"
|
||
assert meta.ceiling_height == 2.50
|
||
assert meta.has_lift is True
|
||
assert meta.total_objects == 12
|
||
assert meta.has_panorama is False
|
||
|
||
|
||
def test_parse_house_meta_no_lift():
|
||
text = "5 объектов Дом 2005 года 16 этажей Монолитное здание 3,00 м потолки"
|
||
meta = YandexValuationScraper._parse_house_meta(text)
|
||
assert meta.has_lift is False
|
||
assert meta.year_built == 2005
|
||
assert meta.total_floors == 16
|
||
assert meta.house_type == "monolith"
|
||
assert meta.ceiling_height == 3.0
|
||
|
||
|
||
def test_parse_house_meta_with_panorama():
|
||
text = "7 объектов Дом 2010 года Панорама Лифт Кирпичное здание"
|
||
meta = YandexValuationScraper._parse_house_meta(text)
|
||
assert meta.has_panorama is True
|
||
assert meta.has_lift is True
|
||
assert meta.house_type == "brick"
|
||
|
||
|
||
def test_house_meta_year_not_present_returns_none():
|
||
text = "8 объектов 5 этажей Панельное здание"
|
||
meta = YandexValuationScraper._parse_house_meta(text)
|
||
assert meta.year_built is None
|
||
assert meta.total_floors == 5
|
||
assert meta.total_objects == 8
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# History item tests
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_history_item_full():
|
||
item = YandexValuationScraper._parse_item_text(_ITEM_1)
|
||
assert item is not None
|
||
assert item.area_m2 == 45.3
|
||
assert item.rooms == 2
|
||
assert item.floor == 4
|
||
assert item.publish_date == date(2023, 3, 15)
|
||
assert item.start_price == 3_200_000
|
||
assert item.last_price == 3_100_000
|
||
assert item.exposure_days == 42
|
||
assert item.status is not None
|
||
assert "продаже" in item.status.lower() or "В" in item.status
|
||
|
||
|
||
def test_parse_history_item_studio_rooms_zero():
|
||
item = YandexValuationScraper._parse_item_text(_ITEM_2)
|
||
assert item is not None
|
||
assert item.rooms == 0 # Studio
|
||
assert item.area_m2 == 25.0
|
||
assert item.floor == 1
|
||
assert item.publish_date == date(2022, 11, 20)
|
||
|
||
|
||
def test_parse_history_item_status_sold():
|
||
item = YandexValuationScraper._parse_item_text(_ITEM_2)
|
||
assert item is not None
|
||
assert item.status is not None
|
||
assert "нят" in item.status.lower() or "Снят" in item.status
|
||
|
||
|
||
def test_parse_history_item_returns_none_for_empty():
|
||
assert YandexValuationScraper._parse_item_text("") is None
|
||
assert YandexValuationScraper._parse_item_text(" ") is None
|
||
|
||
|
||
def test_parse_history_item_returns_none_without_area_or_price():
|
||
# Text too short / no extractable fields
|
||
assert YandexValuationScraper._parse_item_text("нет данных") is None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Chunked text extraction tests
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_items_chunked_dedup():
|
||
"""Duplicate entry (same date + area + floor) must appear only once."""
|
||
duplicate_block = f"{_ITEM_1}\n{_ITEM_1}"
|
||
items = YandexValuationScraper._parse_items_from_chunked_text(duplicate_block)
|
||
# Both items reference 15.03.2023 + 45.3 m2 + floor 4 — must dedup to 1
|
||
matching = [
|
||
i for i in items if i.area_m2 == 45.3 and i.publish_date == date(2023, 3, 15)
|
||
]
|
||
assert len(matching) == 1
|
||
|
||
|
||
def test_parse_items_from_chunked_text_no_dates_returns_empty():
|
||
"""Body text with no DD.MM.YYYY dates should return empty list."""
|
||
items = YandexValuationScraper._parse_items_from_chunked_text(
|
||
"Нет объявлений. Попробуйте изменить параметры поиска."
|
||
)
|
||
assert items == []
|
||
|
||
|
||
def test_parse_items_from_chunked_text_multiple_items():
|
||
"""Three distinct items should be extracted from combined body text."""
|
||
body = f"{_ITEM_1}\n{_ITEM_2}\n{_ITEM_3}"
|
||
items = YandexValuationScraper._parse_items_from_chunked_text(body)
|
||
assert len(items) >= 2 # at minimum items 1 and 2 (item 3 has only start_price)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Full parse round-trip
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_full_result_address_propagated():
|
||
scraper = YandexValuationScraper()
|
||
result = scraper.parse(
|
||
FULL_FIXTURE_HTML,
|
||
address="Екатеринбург, ул. Ленина, 1",
|
||
offer_category="APARTMENT",
|
||
offer_type="SELL",
|
||
page=1,
|
||
source_url="https://realty.yandex.ru/otsenka-kvartiry-po-adresu-onlayn/?page=1",
|
||
)
|
||
assert isinstance(result, YandexValuationResult)
|
||
assert result.address == "Екатеринбург, ул. Ленина, 1"
|
||
assert result.offer_category == "APARTMENT"
|
||
assert result.offer_type == "SELL"
|
||
assert result.page == 1
|
||
assert result.house.year_built == 1981
|
||
assert isinstance(result.history_items, list)
|
||
assert result.raw_payload is not None
|
||
assert "body_len" in result.raw_payload
|
||
|
||
|
||
def test_parse_full_result_house_meta_populated():
|
||
scraper = YandexValuationScraper()
|
||
result = scraper.parse(
|
||
FULL_FIXTURE_HTML,
|
||
address="test",
|
||
offer_category="APARTMENT",
|
||
offer_type="SELL",
|
||
page=1,
|
||
source_url="https://realty.yandex.ru/",
|
||
)
|
||
assert result.house.total_floors == 9
|
||
assert result.house.has_lift is True
|
||
assert result.house.house_type == "panel"
|
||
assert result.house.ceiling_height == 2.5
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# fetch_around raises NotImplementedError
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_raises_not_implemented():
|
||
scraper = YandexValuationScraper()
|
||
with pytest.raises(NotImplementedError):
|
||
await scraper.fetch_around(lat=56.8, lon=60.6)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Ceiling height comma parsing
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_ceiling_height_comma():
|
||
"""Comma decimal separator '2,70 м потолки' must parse to 2.7."""
|
||
text = "Дом 1999 года 5 этажей Кирпичное здание 2,70 м потолки Лифт"
|
||
meta = YandexValuationScraper._parse_house_meta(text)
|
||
assert meta.ceiling_height == 2.7
|