"""Unit tests for YandexValuationScraper — anonymous house-history scraper.
Fixture HTML simulates the Yandex valuation page body text containing:
- House meta block (year, floors, type, ceiling, lift, total objects, panorama)
- 2-3 historical offer entries with full structure
"""
from __future__ import annotations
from datetime import date
import pytest
from app.services.scrapers.yandex_valuation import (
YandexValuationResult,
YandexValuationScraper,
)
# ---------------------------------------------------------------------------
# Fixture helpers
# ---------------------------------------------------------------------------
_HOUSE_META_BLOCK = "12 объектов Дом 1981 года 9 этажей Панельное здание 2,50 м потолки Лифт"
_ITEM_1 = (
"2-комнатная квартира 45,3 м² 4 этаж "
"Опубликовано 15.03.2023 "
"Начальная цена 3 200 000 ₽ 117 000 ₽ за м² "
"Последняя цена 3 100 000 ₽ 114 000 ₽ за м² "
"Длительность экспозиции 42 дня В продаже"
)
_ITEM_2 = (
"Студия 25,0 м² 1 этаж "
"Опубликовано 20.11.2022 "
"Начальная цена 2 100 000 ₽ 84 000 ₽ за м² "
"Последняя цена 1 950 000 ₽ 78 000 ₽ за м² "
"Длительность экспозиции 90 дней Снято"
)
_ITEM_3 = (
"1-комнатная квартира 32,5 м² 7 этаж "
"Опубликовано 01.06.2024 "
"Начальная цена 2 800 000 ₽ "
"Длительность экспозиции 15 дней В продаже"
)
def _make_full_html(body_content: str) -> str:
return f"{body_content}"
FULL_FIXTURE_HTML = _make_full_html(f"{_HOUSE_META_BLOCK}\n{_ITEM_1}\n{_ITEM_2}\n{_ITEM_3}")
# ---------------------------------------------------------------------------
# House meta tests
# ---------------------------------------------------------------------------
def test_parse_house_meta_full():
scraper = YandexValuationScraper()
meta = scraper._parse_house_meta(_HOUSE_META_BLOCK)
assert meta.year_built == 1981
assert meta.total_floors == 9
assert meta.house_type == "panel"
assert meta.ceiling_height == 2.50
assert meta.has_lift is True
assert meta.total_objects == 12
assert meta.has_panorama is False
def test_parse_house_meta_no_lift():
text = "5 объектов Дом 2005 года 16 этажей Монолитное здание 3,00 м потолки"
meta = YandexValuationScraper._parse_house_meta(text)
assert meta.has_lift is False
assert meta.year_built == 2005
assert meta.total_floors == 16
assert meta.house_type == "monolith"
assert meta.ceiling_height == 3.0
def test_parse_house_meta_with_panorama():
text = "7 объектов Дом 2010 года Панорама Лифт Кирпичное здание"
meta = YandexValuationScraper._parse_house_meta(text)
assert meta.has_panorama is True
assert meta.has_lift is True
assert meta.house_type == "brick"
def test_house_meta_year_not_present_returns_none():
text = "8 объектов 5 этажей Панельное здание"
meta = YandexValuationScraper._parse_house_meta(text)
assert meta.year_built is None
assert meta.total_floors == 5
assert meta.total_objects == 8
# ---------------------------------------------------------------------------
# History item tests
# ---------------------------------------------------------------------------
def test_parse_history_item_full():
item = YandexValuationScraper._parse_item_text(_ITEM_1)
assert item is not None
assert item.area_m2 == 45.3
assert item.rooms == 2
assert item.floor == 4
assert item.publish_date == date(2023, 3, 15)
assert item.start_price == 3_200_000
assert item.last_price == 3_100_000
assert item.exposure_days == 42
assert item.status is not None
assert "продаже" in item.status.lower() or "В" in item.status
def test_parse_history_item_studio_rooms_zero():
item = YandexValuationScraper._parse_item_text(_ITEM_2)
assert item is not None
assert item.rooms == 0 # Studio
assert item.area_m2 == 25.0
assert item.floor == 1
assert item.publish_date == date(2022, 11, 20)
def test_parse_history_item_status_sold():
item = YandexValuationScraper._parse_item_text(_ITEM_2)
assert item is not None
assert item.status is not None
assert "нят" in item.status.lower() or "Снят" in item.status
def test_parse_history_item_returns_none_for_empty():
assert YandexValuationScraper._parse_item_text("") is None
assert YandexValuationScraper._parse_item_text(" ") is None
def test_parse_history_item_returns_none_without_area_or_price():
# Text too short / no extractable fields
assert YandexValuationScraper._parse_item_text("нет данных") is None
# ---------------------------------------------------------------------------
# Chunked text extraction tests
# ---------------------------------------------------------------------------
def test_parse_items_chunked_dedup():
"""Duplicate entry (same date + area + floor) must appear only once."""
duplicate_block = f"{_ITEM_1}\n{_ITEM_1}"
items = YandexValuationScraper._parse_items_from_chunked_text(duplicate_block)
# Both items reference 15.03.2023 + 45.3 m2 + floor 4 — must dedup to 1
matching = [i for i in items if i.area_m2 == 45.3 and i.publish_date == date(2023, 3, 15)]
assert len(matching) == 1
def test_parse_items_from_chunked_text_no_dates_returns_empty():
"""Body text with no DD.MM.YYYY dates should return empty list."""
items = YandexValuationScraper._parse_items_from_chunked_text(
"Нет объявлений. Попробуйте изменить параметры поиска."
)
assert items == []
def test_parse_items_from_chunked_text_multiple_items():
"""Three distinct items should be extracted from combined body text."""
body = f"{_ITEM_1}\n{_ITEM_2}\n{_ITEM_3}"
items = YandexValuationScraper._parse_items_from_chunked_text(body)
assert len(items) >= 2 # at minimum items 1 and 2 (item 3 has only start_price)
# ---------------------------------------------------------------------------
# Full parse round-trip
# ---------------------------------------------------------------------------
def test_parse_full_result_address_propagated():
scraper = YandexValuationScraper()
result = scraper.parse(
FULL_FIXTURE_HTML,
address="Екатеринбург, ул. Ленина, 1",
offer_category="APARTMENT",
offer_type="SELL",
page=1,
source_url="https://realty.yandex.ru/otsenka-kvartiry-po-adresu-onlayn/?page=1",
)
assert isinstance(result, YandexValuationResult)
assert result.address == "Екатеринбург, ул. Ленина, 1"
assert result.offer_category == "APARTMENT"
assert result.offer_type == "SELL"
assert result.page == 1
assert result.house.year_built == 1981
assert isinstance(result.history_items, list)
assert result.raw_payload is not None
assert "body_len" in result.raw_payload
def test_parse_full_result_house_meta_populated():
scraper = YandexValuationScraper()
result = scraper.parse(
FULL_FIXTURE_HTML,
address="test",
offer_category="APARTMENT",
offer_type="SELL",
page=1,
source_url="https://realty.yandex.ru/",
)
assert result.house.total_floors == 9
assert result.house.has_lift is True
assert result.house.house_type == "panel"
assert result.house.ceiling_height == 2.5
# ---------------------------------------------------------------------------
# fetch_around raises NotImplementedError
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_fetch_around_raises_not_implemented():
scraper = YandexValuationScraper()
with pytest.raises(NotImplementedError):
await scraper.fetch_around(lat=56.8, lon=60.6)
# ---------------------------------------------------------------------------
# Ceiling height comma parsing
# ---------------------------------------------------------------------------
def test_parse_ceiling_height_comma():
"""Comma decimal separator '2,70 м потолки' must parse to 2.7."""
text = "Дом 1999 года 5 этажей Кирпичное здание 2,70 м потолки Лифт"
meta = YandexValuationScraper._parse_house_meta(text)
assert meta.ceiling_height == 2.7
# ---------------------------------------------------------------------------
# Regression tests — parser fixes (2026-05-24)
# ---------------------------------------------------------------------------
def test_parse_item_extracts_removed_date():
"""Two dates in chunk → first = publish_date, second = removed_date."""
text = "2-комнатная квартира 50,0 м² 3 этаж 10.05.2024 В экспозиции 30 дней 09.06.2024"
item = YandexValuationScraper._parse_item_text(text)
assert item is not None
assert item.publish_date == date(2024, 5, 10)
assert item.removed_date == date(2024, 6, 9)
def test_parse_item_in_sale_has_no_removed_date():
"""Single date + 'В продаже' → removed_date is None."""
text = "1-комнатная квартира 40 м² 5 этаж 10.05.2024 В экспозиции 30 дней В продаже"
item = YandexValuationScraper._parse_item_text(text)
assert item is not None
assert item.publish_date == date(2024, 5, 10)
assert item.removed_date is None
def test_total_floors_extracted_from_dom_meta_not_items():
"""Regression: 'N этаж' inside each item must NOT be matched as dom total_floors.
Real Yandex page has 'M этажей' (plural) in dom-meta and 'N этаж' (singular) per item.
"""
text = (
"Дом 2025 года Панорама 25 этажей Монолитное здание 2,7 м потолки Лифт "
"1-комнатная 40 м² 3 этаж 10.01.2026 В экспозиции 5 дней В продаже "
"2-комнатная 55 м² 17 этаж 05.01.2026 В экспозиции 10 дней В продаже"
)
meta = YandexValuationScraper._parse_house_meta(text)
assert meta.year_built == 2025
assert meta.total_floors == 25
assert meta.house_type == "monolith"
def test_parse_normalizes_nbsp_for_price_per_m2():
"""NBSP (\\xa0) in price tokens must not break price_per_m2 regex.
Real Yandex SSR HTML uses NBSP between digit groups and before ₽.
"""
html = (
""
"Дом 2020 года 16 этажей Монолитное здание "
"2-комнатная 50,0 м² 3 этаж "
"10.05.2024 В экспозиции 30 дней В продаже "
"5\xa0000\xa0000 ₽ 100\xa0000 ₽ за м² "
"4\xa0900\xa0000 ₽ 98\xa0000 ₽ за м² "
""
)
scraper = YandexValuationScraper()
result = scraper.parse(
html,
address="test",
offer_category="APARTMENT",
offer_type="SELL",
page=1,
source_url="http://test/",
)
assert len(result.history_items) >= 1
item = result.history_items[0]
assert item.start_price == 5_000_000
assert item.start_price_per_m2 == 100_000
def test_validate_match_full_match():
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
meta = ValuationHouseMeta(year_built=2025, total_floors=25)
assert meta.validate_match(2025, 25) == 1.0
def test_validate_match_year_mismatch():
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
meta = ValuationHouseMeta(year_built=1984, total_floors=9)
assert meta.validate_match(2025, 25) == 0.0
def test_validate_match_partial():
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
meta = ValuationHouseMeta(year_built=2025, total_floors=10)
# year matches, floors mismatch (>1 tolerance) → 0.5
assert meta.validate_match(2025, 25) == 0.5
def test_validate_match_tolerance_one_year():
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
meta = ValuationHouseMeta(year_built=2024, total_floors=25)
# ±1 year tolerance
assert meta.validate_match(2025, 25) == 1.0
def test_validate_match_no_expected_returns_one():
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
meta = ValuationHouseMeta(year_built=None, total_floors=None)
assert meta.validate_match(None, None) == 1.0
# ---------------------------------------------------------------------------
# Sanity cap regression — area_m2 overflow (2026-05-24)
# ---------------------------------------------------------------------------
def test_parse_item_drops_overflow_area():
"""Regression: area_m2 > 10_000 (e.g. concatenated digits) must be set None,
not blow up the save batch. Dates and exposure_days should survive.
The live bug: chunk-text fallback concatenated digits across a DOM boundary,
producing area_m2=2_025_106.7 → psycopg.NumericValueOutOfRange on INSERT.
The cap drops area_m2 to None so the row still saves with area NULL.
"""
# Bare concatenated digits (no whitespace between groups) — reproduction of
# what the chunked-text path emitted in production. Price is placed after
# the area token so the item stays valid (area_m2 dropped → start_price keeps it).
text = (
"3-комнатная 2025106,7 м² 25 этаж "
"01.07.2024 5 500 000 ₽ В экспозиции 514 дней 26.11.2025"
)
item = YandexValuationScraper._parse_item_text(text)
assert item is not None
assert item.area_m2 is None, f"expected None for absurd area, got {item.area_m2}"
assert item.publish_date == date(2024, 7, 1)
assert item.removed_date == date(2025, 11, 26)
def test_parse_item_keeps_normal_area():
"""Normal residential area must pass through unchanged."""
text = (
"2-комнатная 58,2 м² 3 этаж "
"10.05.2024 9 300 000 ₽ 159 932 ₽ за м² "
"9 300 000 ₽ 159 932 ₽ за м² "
"В экспозиции 30 дней В продаже"
)
item = YandexValuationScraper._parse_item_text(text)
assert item is not None
assert item.area_m2 == 58.2
def test_parse_item_drops_zero_area():
text = "1-комнатная 0 м² 3 этаж 10.05.2024 5 000 000 ₽ В продаже"
item = YandexValuationScraper._parse_item_text(text)
assert item is not None
assert item.area_m2 is None
# ---------------------------------------------------------------------------
# Regex tightening + chronological date ordering (2026-05-24)
# ---------------------------------------------------------------------------
def test_area_regex_rejects_year_concat():
"""RE_ITEM_AREA must NOT match digits glued onto a preceding number.
Real bug on prod: 'опубликовано 01.07.2024' + '52,2 м²' rendered into chunked
text as '202452,2 м²' → matched as 202452.2.
"""
text = "2-комнатная 202452,2 м² 3 этаж 01.07.2024 9 300 000 ₽ В продаже"
item = YandexValuationScraper._parse_item_text(text)
# The "202452,2" token has digits jammed before it (no separator), so the
# tightened regex must NOT match it.
assert item is None or item.area_m2 is None, (
f"expected no area match for concat token, got {item.area_m2 if item else 'None item'}"
)
def test_area_regex_accepts_isolated_token():
"""Normal '58,2 м²' surrounded by whitespace must still be parsed."""
text = "2-комн 58,2 м² 3 этаж 10.05.2024 9 300 000 ₽ В продаже"
item = YandexValuationScraper._parse_item_text(text)
assert item is not None
assert item.area_m2 == 58.2
def test_area_regex_capped_at_4_digits():
"""Anything > 9999.99 m² must not match (real flats are <500 m²)."""
text = "квартира 12345 м² 3 этаж 10.05.2024 5 000 000 ₽ В продаже"
item = YandexValuationScraper._parse_item_text(text)
# 5-digit area shouldn't match; remaining fields still salvageable
assert item is None or item.area_m2 is None
def test_dates_sorted_publish_before_removed():
"""Even if page-text emits removed-date first, parser must put earlier
date in publish_date and later in removed_date.
"""
# Reversed order: removed appears in text before publish
text = (
"2-комн 50,0 м² 3 этаж "
"13.08.2025 В экспозиции 156 дней 12.02.2026 "
"8 000 000 ₽ В продаже"
)
item = YandexValuationScraper._parse_item_text(text)
assert item is not None
from datetime import date as _d
assert item.publish_date == _d(2025, 8, 13)
assert item.removed_date == _d(2026, 2, 12)
def test_single_date_is_publish_only():
"""One date → publish_date set, removed_date None."""
text = "1-комн 40,0 м² 3 этаж 15.03.2024 В экспозиции 30 дней В продаже"
item = YandexValuationScraper._parse_item_text(text)
assert item is not None
from datetime import date as _d
assert item.publish_date == _d(2024, 3, 15)
assert item.removed_date is None
def test_area_regex_two_komn_chunk():
"""2-комн chunks were 100% rejected by PR #541 lookbehind. Real bug from
Базовый 52 sweep: 'rooms=2 → area_m2 is None' for every 2-комн row.
"""
# Approximate the chunked text Yandex emits for a 2-room item
text = "8 000 000 ₽ за м²2-комнатная 52,2 м² 5 этаж 10.05.2024 8 000 000 ₽ В продаже"
item = YandexValuationScraper._parse_item_text(text)
assert item is not None
assert item.area_m2 == 52.2, (
f"2-комн area must parse despite preceding tokens, got {item.area_m2}"
)
def test_area_regex_still_blocks_year_concat():
"""The relaxed regex must still reject year+area concat."""
# '2024' directly fused to '42,6 м²' as can happen after DOM strip
text = "квартира 202442,6 м² 3 этаж 10.05.2024 9 000 000 ₽ В продаже"
item = YandexValuationScraper._parse_item_text(text)
# The '202442,6' token: a `\\d{2,4}` greedy match starting at first '2'
# gives '2024', then needs '[.,]\\d{1,2}' but '4' follows → no match.
# Any internal start is blocked by `(?
48,5 м², 1-комнатная19 этаж
5,1 млн ₽105 155 ₽ за м²
5,1 млн ₽105 155 ₽ за м²
23.10.2023В экспозиции 945 дней
В продаже
58,2 м², 2-комнатная3 этаж
9,3 млн ₽159 932 ₽ за м²
8,9 млн ₽152 062 ₽ за м²
12.02.2026В экспозиции 156 дней
17.07.2026
"""
def test_dom_parser_extracts_two_rows():
scraper = YandexValuationScraper()
house_meta = "Дом 2020 года 25 этажей Монолитное здание "
result = scraper.parse(
f"{house_meta}{_DOM_FIXTURE_ROW_HTML}",
address="test",
offer_category="APARTMENT",
offer_type="SELL",
page=1,
source_url="http://test/",
)
assert len(result.history_items) == 2
# Row 1 — 1-комн in sale
r1 = result.history_items[0]
assert r1.area_m2 == 48.5
assert r1.rooms == 1
assert r1.floor == 19
assert r1.start_price == 5_100_000
assert r1.start_price_per_m2 == 105_155
assert r1.last_price == 5_100_000
assert r1.publish_date == date(2023, 10, 23)
assert r1.exposure_days == 945
assert r1.removed_date is None
assert r1.status == "В продаже"
# Row 2 — 2-комн sold with both dates
r2 = result.history_items[1]
assert r2.area_m2 == 58.2
assert r2.rooms == 2
assert r2.floor == 3
assert r2.start_price == 9_300_000
assert r2.last_price == 8_900_000
assert r2.publish_date == date(2026, 2, 12)
assert r2.removed_date == date(2026, 7, 17)
assert r2.exposure_days == 156
def test_dom_parser_studio_row():
html = (
""
''
'
'
'
Студия 25 м²5 этаж
'
'
4 млн ₽160 000 ₽ за м²
'
'
4 млн ₽160 000 ₽ за м²
'
'
01.03.2024В экспозиции 30 дней
'
'
В продаже
'
"
'
'
48,5 м²
'
'
19 этаж
'
"