4939d04360
Live observation: sweep on "Базовый переулок, 52" produced area_m2=2_025_106.7 (concatenated digits across DOM boundaries) -> psycopg.NumericValueOutOfRange on INSERT, lost entire batch of 30 items from page=1. Cap at 10_000 m2 (residential ceiling) and drop <=0. Keeps the rest of the item (price, dates, status) so the row still persists with area_m2=NULL -- better than losing the whole batch.
386 lines
15 KiB
Python
386 lines
15 KiB
Python
"""Unit tests for YandexValuationScraper — anonymous house-history scraper.
|
||
|
||
Fixture HTML simulates the Yandex valuation page body text containing:
|
||
- House meta block (year, floors, type, ceiling, lift, total objects, panorama)
|
||
- 2-3 historical offer entries with full structure
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from datetime import date
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers.yandex_valuation import (
|
||
YandexValuationResult,
|
||
YandexValuationScraper,
|
||
)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Fixture helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_HOUSE_META_BLOCK = "12 объектов Дом 1981 года 9 этажей Панельное здание 2,50 м потолки Лифт"
|
||
|
||
_ITEM_1 = (
|
||
"2-комнатная квартира 45,3 м² 4 этаж "
|
||
"Опубликовано 15.03.2023 "
|
||
"Начальная цена 3 200 000 ₽ 117 000 ₽ за м² "
|
||
"Последняя цена 3 100 000 ₽ 114 000 ₽ за м² "
|
||
"Длительность экспозиции 42 дня В продаже"
|
||
)
|
||
|
||
_ITEM_2 = (
|
||
"Студия 25,0 м² 1 этаж "
|
||
"Опубликовано 20.11.2022 "
|
||
"Начальная цена 2 100 000 ₽ 84 000 ₽ за м² "
|
||
"Последняя цена 1 950 000 ₽ 78 000 ₽ за м² "
|
||
"Длительность экспозиции 90 дней Снято"
|
||
)
|
||
|
||
_ITEM_3 = (
|
||
"1-комнатная квартира 32,5 м² 7 этаж "
|
||
"Опубликовано 01.06.2024 "
|
||
"Начальная цена 2 800 000 ₽ "
|
||
"Длительность экспозиции 15 дней В продаже"
|
||
)
|
||
|
||
|
||
def _make_full_html(body_content: str) -> str:
|
||
return f"<html><body>{body_content}</body></html>"
|
||
|
||
|
||
FULL_FIXTURE_HTML = _make_full_html(f"{_HOUSE_META_BLOCK}\n{_ITEM_1}\n{_ITEM_2}\n{_ITEM_3}")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# House meta tests
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_house_meta_full():
|
||
scraper = YandexValuationScraper()
|
||
meta = scraper._parse_house_meta(_HOUSE_META_BLOCK)
|
||
assert meta.year_built == 1981
|
||
assert meta.total_floors == 9
|
||
assert meta.house_type == "panel"
|
||
assert meta.ceiling_height == 2.50
|
||
assert meta.has_lift is True
|
||
assert meta.total_objects == 12
|
||
assert meta.has_panorama is False
|
||
|
||
|
||
def test_parse_house_meta_no_lift():
|
||
text = "5 объектов Дом 2005 года 16 этажей Монолитное здание 3,00 м потолки"
|
||
meta = YandexValuationScraper._parse_house_meta(text)
|
||
assert meta.has_lift is False
|
||
assert meta.year_built == 2005
|
||
assert meta.total_floors == 16
|
||
assert meta.house_type == "monolith"
|
||
assert meta.ceiling_height == 3.0
|
||
|
||
|
||
def test_parse_house_meta_with_panorama():
|
||
text = "7 объектов Дом 2010 года Панорама Лифт Кирпичное здание"
|
||
meta = YandexValuationScraper._parse_house_meta(text)
|
||
assert meta.has_panorama is True
|
||
assert meta.has_lift is True
|
||
assert meta.house_type == "brick"
|
||
|
||
|
||
def test_house_meta_year_not_present_returns_none():
|
||
text = "8 объектов 5 этажей Панельное здание"
|
||
meta = YandexValuationScraper._parse_house_meta(text)
|
||
assert meta.year_built is None
|
||
assert meta.total_floors == 5
|
||
assert meta.total_objects == 8
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# History item tests
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_history_item_full():
|
||
item = YandexValuationScraper._parse_item_text(_ITEM_1)
|
||
assert item is not None
|
||
assert item.area_m2 == 45.3
|
||
assert item.rooms == 2
|
||
assert item.floor == 4
|
||
assert item.publish_date == date(2023, 3, 15)
|
||
assert item.start_price == 3_200_000
|
||
assert item.last_price == 3_100_000
|
||
assert item.exposure_days == 42
|
||
assert item.status is not None
|
||
assert "продаже" in item.status.lower() or "В" in item.status
|
||
|
||
|
||
def test_parse_history_item_studio_rooms_zero():
|
||
item = YandexValuationScraper._parse_item_text(_ITEM_2)
|
||
assert item is not None
|
||
assert item.rooms == 0 # Studio
|
||
assert item.area_m2 == 25.0
|
||
assert item.floor == 1
|
||
assert item.publish_date == date(2022, 11, 20)
|
||
|
||
|
||
def test_parse_history_item_status_sold():
|
||
item = YandexValuationScraper._parse_item_text(_ITEM_2)
|
||
assert item is not None
|
||
assert item.status is not None
|
||
assert "нят" in item.status.lower() or "Снят" in item.status
|
||
|
||
|
||
def test_parse_history_item_returns_none_for_empty():
|
||
assert YandexValuationScraper._parse_item_text("") is None
|
||
assert YandexValuationScraper._parse_item_text(" ") is None
|
||
|
||
|
||
def test_parse_history_item_returns_none_without_area_or_price():
|
||
# Text too short / no extractable fields
|
||
assert YandexValuationScraper._parse_item_text("нет данных") is None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Chunked text extraction tests
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_items_chunked_dedup():
|
||
"""Duplicate entry (same date + area + floor) must appear only once."""
|
||
duplicate_block = f"{_ITEM_1}\n{_ITEM_1}"
|
||
items = YandexValuationScraper._parse_items_from_chunked_text(duplicate_block)
|
||
# Both items reference 15.03.2023 + 45.3 m2 + floor 4 — must dedup to 1
|
||
matching = [i for i in items if i.area_m2 == 45.3 and i.publish_date == date(2023, 3, 15)]
|
||
assert len(matching) == 1
|
||
|
||
|
||
def test_parse_items_from_chunked_text_no_dates_returns_empty():
|
||
"""Body text with no DD.MM.YYYY dates should return empty list."""
|
||
items = YandexValuationScraper._parse_items_from_chunked_text(
|
||
"Нет объявлений. Попробуйте изменить параметры поиска."
|
||
)
|
||
assert items == []
|
||
|
||
|
||
def test_parse_items_from_chunked_text_multiple_items():
|
||
"""Three distinct items should be extracted from combined body text."""
|
||
body = f"{_ITEM_1}\n{_ITEM_2}\n{_ITEM_3}"
|
||
items = YandexValuationScraper._parse_items_from_chunked_text(body)
|
||
assert len(items) >= 2 # at minimum items 1 and 2 (item 3 has only start_price)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Full parse round-trip
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_full_result_address_propagated():
|
||
scraper = YandexValuationScraper()
|
||
result = scraper.parse(
|
||
FULL_FIXTURE_HTML,
|
||
address="Екатеринбург, ул. Ленина, 1",
|
||
offer_category="APARTMENT",
|
||
offer_type="SELL",
|
||
page=1,
|
||
source_url="https://realty.yandex.ru/otsenka-kvartiry-po-adresu-onlayn/?page=1",
|
||
)
|
||
assert isinstance(result, YandexValuationResult)
|
||
assert result.address == "Екатеринбург, ул. Ленина, 1"
|
||
assert result.offer_category == "APARTMENT"
|
||
assert result.offer_type == "SELL"
|
||
assert result.page == 1
|
||
assert result.house.year_built == 1981
|
||
assert isinstance(result.history_items, list)
|
||
assert result.raw_payload is not None
|
||
assert "body_len" in result.raw_payload
|
||
|
||
|
||
def test_parse_full_result_house_meta_populated():
|
||
scraper = YandexValuationScraper()
|
||
result = scraper.parse(
|
||
FULL_FIXTURE_HTML,
|
||
address="test",
|
||
offer_category="APARTMENT",
|
||
offer_type="SELL",
|
||
page=1,
|
||
source_url="https://realty.yandex.ru/",
|
||
)
|
||
assert result.house.total_floors == 9
|
||
assert result.house.has_lift is True
|
||
assert result.house.house_type == "panel"
|
||
assert result.house.ceiling_height == 2.5
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# fetch_around raises NotImplementedError
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_raises_not_implemented():
|
||
scraper = YandexValuationScraper()
|
||
with pytest.raises(NotImplementedError):
|
||
await scraper.fetch_around(lat=56.8, lon=60.6)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Ceiling height comma parsing
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_ceiling_height_comma():
|
||
"""Comma decimal separator '2,70 м потолки' must parse to 2.7."""
|
||
text = "Дом 1999 года 5 этажей Кирпичное здание 2,70 м потолки Лифт"
|
||
meta = YandexValuationScraper._parse_house_meta(text)
|
||
assert meta.ceiling_height == 2.7
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Regression tests — parser fixes (2026-05-24)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_item_extracts_removed_date():
|
||
"""Two dates in chunk → first = publish_date, second = removed_date."""
|
||
text = "2-комнатная квартира 50,0 м² 3 этаж 10.05.2024 В экспозиции 30 дней 09.06.2024"
|
||
item = YandexValuationScraper._parse_item_text(text)
|
||
assert item is not None
|
||
assert item.publish_date == date(2024, 5, 10)
|
||
assert item.removed_date == date(2024, 6, 9)
|
||
|
||
|
||
def test_parse_item_in_sale_has_no_removed_date():
|
||
"""Single date + 'В продаже' → removed_date is None."""
|
||
text = "1-комнатная квартира 40 м² 5 этаж 10.05.2024 В экспозиции 30 дней В продаже"
|
||
item = YandexValuationScraper._parse_item_text(text)
|
||
assert item is not None
|
||
assert item.publish_date == date(2024, 5, 10)
|
||
assert item.removed_date is None
|
||
|
||
|
||
def test_total_floors_extracted_from_dom_meta_not_items():
|
||
"""Regression: 'N этаж' inside each item must NOT be matched as dom total_floors.
|
||
Real Yandex page has 'M этажей' (plural) in dom-meta and 'N этаж' (singular) per item.
|
||
"""
|
||
text = (
|
||
"Дом 2025 года Панорама 25 этажей Монолитное здание 2,7 м потолки Лифт "
|
||
"1-комнатная 40 м² 3 этаж 10.01.2026 В экспозиции 5 дней В продаже "
|
||
"2-комнатная 55 м² 17 этаж 05.01.2026 В экспозиции 10 дней В продаже"
|
||
)
|
||
meta = YandexValuationScraper._parse_house_meta(text)
|
||
assert meta.year_built == 2025
|
||
assert meta.total_floors == 25
|
||
assert meta.house_type == "monolith"
|
||
|
||
|
||
def test_parse_normalizes_nbsp_for_price_per_m2():
|
||
"""NBSP (\\xa0) in price tokens must not break price_per_m2 regex.
|
||
Real Yandex SSR HTML uses NBSP between digit groups and before ₽.
|
||
"""
|
||
html = (
|
||
"<html><body>"
|
||
"Дом 2020 года 16 этажей Монолитное здание "
|
||
"2-комнатная 50,0 м² 3 этаж "
|
||
"10.05.2024 В экспозиции 30 дней В продаже "
|
||
"5\xa0000\xa0000 ₽ 100\xa0000 ₽ за м² "
|
||
"4\xa0900\xa0000 ₽ 98\xa0000 ₽ за м² "
|
||
"</body></html>"
|
||
)
|
||
scraper = YandexValuationScraper()
|
||
result = scraper.parse(
|
||
html,
|
||
address="test",
|
||
offer_category="APARTMENT",
|
||
offer_type="SELL",
|
||
page=1,
|
||
source_url="http://test/",
|
||
)
|
||
assert len(result.history_items) >= 1
|
||
item = result.history_items[0]
|
||
assert item.start_price == 5_000_000
|
||
assert item.start_price_per_m2 == 100_000
|
||
|
||
|
||
def test_validate_match_full_match():
|
||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||
|
||
meta = ValuationHouseMeta(year_built=2025, total_floors=25)
|
||
assert meta.validate_match(2025, 25) == 1.0
|
||
|
||
|
||
def test_validate_match_year_mismatch():
|
||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||
|
||
meta = ValuationHouseMeta(year_built=1984, total_floors=9)
|
||
assert meta.validate_match(2025, 25) == 0.0
|
||
|
||
|
||
def test_validate_match_partial():
|
||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||
|
||
meta = ValuationHouseMeta(year_built=2025, total_floors=10)
|
||
# year matches, floors mismatch (>1 tolerance) → 0.5
|
||
assert meta.validate_match(2025, 25) == 0.5
|
||
|
||
|
||
def test_validate_match_tolerance_one_year():
|
||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||
|
||
meta = ValuationHouseMeta(year_built=2024, total_floors=25)
|
||
# ±1 year tolerance
|
||
assert meta.validate_match(2025, 25) == 1.0
|
||
|
||
|
||
def test_validate_match_no_expected_returns_one():
|
||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||
|
||
meta = ValuationHouseMeta(year_built=None, total_floors=None)
|
||
assert meta.validate_match(None, None) == 1.0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Sanity cap regression — area_m2 overflow (2026-05-24)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_item_drops_overflow_area():
|
||
"""Regression: area_m2 > 10_000 (e.g. concatenated digits) must be set None,
|
||
not blow up the save batch. Dates and exposure_days should survive.
|
||
|
||
The live bug: chunk-text fallback concatenated digits across a DOM boundary,
|
||
producing area_m2=2_025_106.7 → psycopg.NumericValueOutOfRange on INSERT.
|
||
The cap drops area_m2 to None so the row still saves with area NULL.
|
||
"""
|
||
# Bare concatenated digits (no whitespace between groups) — reproduction of
|
||
# what the chunked-text path emitted in production. Price is placed after
|
||
# the area token so the item stays valid (area_m2 dropped → start_price keeps it).
|
||
text = (
|
||
"3-комнатная 2025106,7 м² 25 этаж "
|
||
"01.07.2024 5 500 000 ₽ В экспозиции 514 дней 26.11.2025"
|
||
)
|
||
item = YandexValuationScraper._parse_item_text(text)
|
||
assert item is not None
|
||
assert item.area_m2 is None, f"expected None for absurd area, got {item.area_m2}"
|
||
assert item.publish_date == date(2024, 7, 1)
|
||
assert item.removed_date == date(2025, 11, 26)
|
||
|
||
|
||
def test_parse_item_keeps_normal_area():
|
||
"""Normal residential area must pass through unchanged."""
|
||
text = (
|
||
"2-комнатная 58,2 м² 3 этаж "
|
||
"10.05.2024 9 300 000 ₽ 159 932 ₽ за м² "
|
||
"9 300 000 ₽ 159 932 ₽ за м² "
|
||
"В экспозиции 30 дней В продаже"
|
||
)
|
||
item = YandexValuationScraper._parse_item_text(text)
|
||
assert item is not None
|
||
assert item.area_m2 == 58.2
|
||
|
||
|
||
def test_parse_item_drops_zero_area():
|
||
text = "1-комнатная 0 м² 3 этаж 10.05.2024 5 000 000 ₽ В продаже"
|
||
item = YandexValuationScraper._parse_item_text(text)
|
||
assert item is not None
|
||
assert item.area_m2 is None
|