a9b87aa311
All checks were successful
CI Trade-In / changes (pull_request) Successful in 8s
CI / changes (pull_request) Successful in 8s
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
CI Trade-In / backend-tests (pull_request) Successful in 1m36s
Удалены (0 runtime importers в app/, scripts/, packages/; проверено grep'ом, включая intra-legacy импорты в оставшихся 21 legacy-модуле scrapers/): - backend/app/services/scrapers/avito_imv.py - backend/app/services/scrapers/domclick_detail.py - backend/app/services/scrapers/ekb_geoportal_client.py - backend/app/services/scrapers/yandex_detail.py - backend/app/services/scrapers/yandex_valuation.py Тесты — удалены (pure legacy-vs-kit parity, kit-only coverage уже есть в другом месте): - tests/scrapers/test_ekb_geoportal_client_kit_parity.py - tests/scrapers/test_domclick_detail_kit_parity.py Тесты — конвертированы в kit-only (импорт легаси заменён на scraper_kit.*, покрытие сохранено без потерь): - tests/scrapers/test_avito_imv_kit_parity.py (parity-тесты убраны, config-footgun regression на kit-стороне оставлен) - tests/scrapers/test_avito_imv_browser_transport.py - tests/scrapers/test_yandex_valuation_kit_migration.py (parity убрана, footgun- регрессия mandatory config/delay_provider оставлена) - tests/scrapers/test_domclick_detail.py (+ exceptions переведены на scraper_kit.domclick_exceptions, т.к. kit parse_detail_html поднимает их) - tests/test_avito_imv_parse.py - tests/test_ekb_geoportal_ingest.py (client-часть; ingest/geocoder-часть не трогали — не зависят от удалённого модуля) - tests/test_yandex_detail.py - tests/test_yandex_detail_structural.py - tests/test_yandex_valuation.py (YandexValuationScraper() → config=_KIT_CONFIG, kit конструктор требует обязательный config) - tests/test_yandex_valuation_save.py - tests/test_extval_house_id_write_path.py (только yandex_valuation-часть; cian_valuation остался нетронутым — живой легаси-модуль) - tests/test_yandex_history_area_filter.py Тесты — частично отредактированы (убраны только части про удалённые модули, живые легаси-модули/их parity не тронуты): - tests/scrapers/test_admin_domclick_ingest_kit_parity.py (base.py/ScrapedLot parity остался) - tests/scrapers/test_admin_yandex_kit_parity.py (yandex_realty parity остался) - tests/scrapers/test_admin_avito_kit_parity.py (avito.py/avito_houses.py parity остался; _parse_price parity убран) - tests/scrapers/test_avito_unix_date_tz_consistency.py (IMV-ветка переведена на kit avito.imv, легаси avito/avito_houses/avito_shared/avito_detail не тронуты) - tests/tasks/test_yandex_detail_backfill.py (subject — kit-задача, уже вызывает scraper_kit.providers.yandex.detail; save_detail_enrichment coverage-тесты внизу файла переведены на kit) - tests/test_scraper_kit_yandex_golden_parity.py (detail/valuation parity убраны; serp/newbuilding/house_type_normalizer/build_url для yandex_realty/ yandex_newbuilding не тронуты) - tests/test_scraper_proxy.py (YandexValuation proxy-тесты переведены на kit config= DI; test_avito_imv_own_session_receives_proxies удалён — дублирует test_avito_imv_kit_parity.py footgun-тесты) - tests/test_yandex_scrapers_delay_wiring.py (yandex_detail/yandex_valuation → kit delay_provider=; yandex_realty/yandex_newbuilding не тронуты) - tests/test_scraper_kit_group_c_backfill_kit_parity.py (docstring уточнение, без функциональных изменений) Полный backend-suite зелёный (3253 passed, 6 skipped) кроме известного pre-existing flake test_search_api.py::test_search_cache_hit (#2208, не регрессия этого PR). ruff check + ruff format — чисто на всех изменённых файлах.
636 lines
25 KiB
Python
636 lines
25 KiB
Python
"""Unit tests for YandexValuationScraper — anonymous house-history scraper.
|
||
|
||
Fixture HTML simulates the Yandex valuation page body text containing:
|
||
- House meta block (year, floors, type, ceiling, lift, total objects, panorama)
|
||
- 2-3 historical offer entries with full structure
|
||
|
||
Легаси `app.services.scrapers.yandex_valuation` удалён (#2277 финальный шаг
|
||
scraper_kit-миграции, 0 runtime importers) — тесты переведены на kit-эквивалент
|
||
`scraper_kit.providers.yandex.valuation` без изменения покрытия. Kit-конструктор
|
||
требует обязательный `config:` (см. `tests/scrapers/test_yandex_valuation_kit_migration.py`
|
||
для footgun-регрессии) — используем тот же duck-typed `SimpleNamespace`, что и
|
||
`tests/test_scraper_kit_yandex_golden_parity.py`.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from datetime import date
|
||
from types import SimpleNamespace
|
||
|
||
import pytest
|
||
from scraper_kit.providers.yandex.valuation import (
|
||
YandexValuationResult,
|
||
YandexValuationScraper,
|
||
)
|
||
|
||
_KIT_CONFIG = SimpleNamespace(
|
||
scraper_proxy_url=None,
|
||
yandex_proxy_rotate_url=None,
|
||
avito_proxy_rotate_url=None,
|
||
)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Fixture helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_HOUSE_META_BLOCK = "12 объектов Дом 1981 года 9 этажей Панельное здание 2,50 м потолки Лифт"
|
||
|
||
_ITEM_1 = (
|
||
"2-комнатная квартира 45,3 м² 4 этаж "
|
||
"Опубликовано 15.03.2023 "
|
||
"Начальная цена 3 200 000 ₽ 117 000 ₽ за м² "
|
||
"Последняя цена 3 100 000 ₽ 114 000 ₽ за м² "
|
||
"Длительность экспозиции 42 дня В продаже"
|
||
)
|
||
|
||
_ITEM_2 = (
|
||
"Студия 25,0 м² 1 этаж "
|
||
"Опубликовано 20.11.2022 "
|
||
"Начальная цена 2 100 000 ₽ 84 000 ₽ за м² "
|
||
"Последняя цена 1 950 000 ₽ 78 000 ₽ за м² "
|
||
"Длительность экспозиции 90 дней Снято"
|
||
)
|
||
|
||
_ITEM_3 = (
|
||
"1-комнатная квартира 32,5 м² 7 этаж "
|
||
"Опубликовано 01.06.2024 "
|
||
"Начальная цена 2 800 000 ₽ "
|
||
"Длительность экспозиции 15 дней В продаже"
|
||
)
|
||
|
||
|
||
def _make_full_html(body_content: str) -> str:
|
||
return f"<html><body>{body_content}</body></html>"
|
||
|
||
|
||
FULL_FIXTURE_HTML = _make_full_html(f"{_HOUSE_META_BLOCK}\n{_ITEM_1}\n{_ITEM_2}\n{_ITEM_3}")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# House meta tests
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_house_meta_full():
|
||
scraper = YandexValuationScraper(config=_KIT_CONFIG)
|
||
meta = scraper._parse_house_meta(_HOUSE_META_BLOCK)
|
||
assert meta.year_built == 1981
|
||
assert meta.total_floors == 9
|
||
assert meta.house_type == "panel"
|
||
assert meta.ceiling_height == 2.50
|
||
assert meta.has_lift is True
|
||
assert meta.total_objects == 12
|
||
assert meta.has_panorama is False
|
||
|
||
|
||
def test_parse_house_meta_no_lift():
|
||
text = "5 объектов Дом 2005 года 16 этажей Монолитное здание 3,00 м потолки"
|
||
meta = YandexValuationScraper._parse_house_meta(text)
|
||
assert meta.has_lift is False
|
||
assert meta.year_built == 2005
|
||
assert meta.total_floors == 16
|
||
assert meta.house_type == "monolith"
|
||
assert meta.ceiling_height == 3.0
|
||
|
||
|
||
def test_parse_house_meta_with_panorama():
|
||
text = "7 объектов Дом 2010 года Панорама Лифт Кирпичное здание"
|
||
meta = YandexValuationScraper._parse_house_meta(text)
|
||
assert meta.has_panorama is True
|
||
assert meta.has_lift is True
|
||
assert meta.house_type == "brick"
|
||
|
||
|
||
def test_house_meta_year_not_present_returns_none():
|
||
text = "8 объектов 5 этажей Панельное здание"
|
||
meta = YandexValuationScraper._parse_house_meta(text)
|
||
assert meta.year_built is None
|
||
assert meta.total_floors == 5
|
||
assert meta.total_objects == 8
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# History item tests
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_history_item_full():
|
||
item = YandexValuationScraper._parse_item_text(_ITEM_1)
|
||
assert item is not None
|
||
assert item.area_m2 == 45.3
|
||
assert item.rooms == 2
|
||
assert item.floor == 4
|
||
assert item.publish_date == date(2023, 3, 15)
|
||
assert item.start_price == 3_200_000
|
||
assert item.last_price == 3_100_000
|
||
assert item.exposure_days == 42
|
||
assert item.status is not None
|
||
assert "продаже" in item.status.lower() or "В" in item.status
|
||
|
||
|
||
def test_parse_history_item_studio_rooms_zero():
|
||
item = YandexValuationScraper._parse_item_text(_ITEM_2)
|
||
assert item is not None
|
||
assert item.rooms == 0 # Studio
|
||
assert item.area_m2 == 25.0
|
||
assert item.floor == 1
|
||
assert item.publish_date == date(2022, 11, 20)
|
||
|
||
|
||
def test_parse_history_item_status_sold():
|
||
item = YandexValuationScraper._parse_item_text(_ITEM_2)
|
||
assert item is not None
|
||
assert item.status is not None
|
||
assert "нят" in item.status.lower() or "Снят" in item.status
|
||
|
||
|
||
def test_parse_history_item_returns_none_for_empty():
|
||
assert YandexValuationScraper._parse_item_text("") is None
|
||
assert YandexValuationScraper._parse_item_text(" ") is None
|
||
|
||
|
||
def test_parse_history_item_returns_none_without_area_or_price():
|
||
# Text too short / no extractable fields
|
||
assert YandexValuationScraper._parse_item_text("нет данных") is None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Chunked text extraction tests
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_items_chunked_dedup():
|
||
"""Duplicate entry (same date + area + floor) must appear only once."""
|
||
duplicate_block = f"{_ITEM_1}\n{_ITEM_1}"
|
||
items = YandexValuationScraper._parse_items_from_chunked_text(duplicate_block)
|
||
# Both items reference 15.03.2023 + 45.3 m2 + floor 4 — must dedup to 1
|
||
matching = [i for i in items if i.area_m2 == 45.3 and i.publish_date == date(2023, 3, 15)]
|
||
assert len(matching) == 1
|
||
|
||
|
||
def test_parse_items_from_chunked_text_no_dates_returns_empty():
|
||
"""Body text with no DD.MM.YYYY dates should return empty list."""
|
||
items = YandexValuationScraper._parse_items_from_chunked_text(
|
||
"Нет объявлений. Попробуйте изменить параметры поиска."
|
||
)
|
||
assert items == []
|
||
|
||
|
||
def test_parse_items_from_chunked_text_multiple_items():
|
||
"""Three distinct items should be extracted from combined body text."""
|
||
body = f"{_ITEM_1}\n{_ITEM_2}\n{_ITEM_3}"
|
||
items = YandexValuationScraper._parse_items_from_chunked_text(body)
|
||
assert len(items) >= 2 # at minimum items 1 and 2 (item 3 has only start_price)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Full parse round-trip
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_full_result_address_propagated():
|
||
scraper = YandexValuationScraper(config=_KIT_CONFIG)
|
||
result = scraper.parse(
|
||
FULL_FIXTURE_HTML,
|
||
address="Екатеринбург, ул. Ленина, 1",
|
||
offer_category="APARTMENT",
|
||
offer_type="SELL",
|
||
page=1,
|
||
source_url="https://realty.yandex.ru/otsenka-kvartiry-po-adresu-onlayn/?page=1",
|
||
)
|
||
assert isinstance(result, YandexValuationResult)
|
||
assert result.address == "Екатеринбург, ул. Ленина, 1"
|
||
assert result.offer_category == "APARTMENT"
|
||
assert result.offer_type == "SELL"
|
||
assert result.page == 1
|
||
assert result.house.year_built == 1981
|
||
assert isinstance(result.history_items, list)
|
||
assert result.raw_payload is not None
|
||
assert "body_len" in result.raw_payload
|
||
|
||
|
||
def test_parse_full_result_house_meta_populated():
|
||
scraper = YandexValuationScraper(config=_KIT_CONFIG)
|
||
result = scraper.parse(
|
||
FULL_FIXTURE_HTML,
|
||
address="test",
|
||
offer_category="APARTMENT",
|
||
offer_type="SELL",
|
||
page=1,
|
||
source_url="https://realty.yandex.ru/",
|
||
)
|
||
assert result.house.total_floors == 9
|
||
assert result.house.has_lift is True
|
||
assert result.house.house_type == "panel"
|
||
assert result.house.ceiling_height == 2.5
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# fetch_around raises NotImplementedError
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_raises_not_implemented():
|
||
scraper = YandexValuationScraper(config=_KIT_CONFIG)
|
||
with pytest.raises(NotImplementedError):
|
||
await scraper.fetch_around(lat=56.8, lon=60.6)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Ceiling height comma parsing
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_ceiling_height_comma():
|
||
"""Comma decimal separator '2,70 м потолки' must parse to 2.7."""
|
||
text = "Дом 1999 года 5 этажей Кирпичное здание 2,70 м потолки Лифт"
|
||
meta = YandexValuationScraper._parse_house_meta(text)
|
||
assert meta.ceiling_height == 2.7
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Regression tests — parser fixes (2026-05-24)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_item_extracts_removed_date():
|
||
"""Two dates in chunk → first = publish_date, second = removed_date."""
|
||
text = "2-комнатная квартира 50,0 м² 3 этаж 10.05.2024 В экспозиции 30 дней 09.06.2024"
|
||
item = YandexValuationScraper._parse_item_text(text)
|
||
assert item is not None
|
||
assert item.publish_date == date(2024, 5, 10)
|
||
assert item.removed_date == date(2024, 6, 9)
|
||
|
||
|
||
def test_parse_item_in_sale_has_no_removed_date():
|
||
"""Single date + 'В продаже' → removed_date is None."""
|
||
text = "1-комнатная квартира 40 м² 5 этаж 10.05.2024 В экспозиции 30 дней В продаже"
|
||
item = YandexValuationScraper._parse_item_text(text)
|
||
assert item is not None
|
||
assert item.publish_date == date(2024, 5, 10)
|
||
assert item.removed_date is None
|
||
|
||
|
||
def test_total_floors_extracted_from_dom_meta_not_items():
|
||
"""Regression: 'N этаж' inside each item must NOT be matched as dom total_floors.
|
||
Real Yandex page has 'M этажей' (plural) in dom-meta and 'N этаж' (singular) per item.
|
||
"""
|
||
text = (
|
||
"Дом 2025 года Панорама 25 этажей Монолитное здание 2,7 м потолки Лифт "
|
||
"1-комнатная 40 м² 3 этаж 10.01.2026 В экспозиции 5 дней В продаже "
|
||
"2-комнатная 55 м² 17 этаж 05.01.2026 В экспозиции 10 дней В продаже"
|
||
)
|
||
meta = YandexValuationScraper._parse_house_meta(text)
|
||
assert meta.year_built == 2025
|
||
assert meta.total_floors == 25
|
||
assert meta.house_type == "monolith"
|
||
|
||
|
||
def test_parse_normalizes_nbsp_for_price_per_m2():
|
||
"""NBSP (\\xa0) in price tokens must not break price_per_m2 regex.
|
||
Real Yandex SSR HTML uses NBSP between digit groups and before ₽.
|
||
"""
|
||
html = (
|
||
"<html><body>"
|
||
"Дом 2020 года 16 этажей Монолитное здание "
|
||
"2-комнатная 50,0 м² 3 этаж "
|
||
"10.05.2024 В экспозиции 30 дней В продаже "
|
||
"5\xa0000\xa0000 ₽ 100\xa0000 ₽ за м² "
|
||
"4\xa0900\xa0000 ₽ 98\xa0000 ₽ за м² "
|
||
"</body></html>"
|
||
)
|
||
scraper = YandexValuationScraper(config=_KIT_CONFIG)
|
||
result = scraper.parse(
|
||
html,
|
||
address="test",
|
||
offer_category="APARTMENT",
|
||
offer_type="SELL",
|
||
page=1,
|
||
source_url="http://test/",
|
||
)
|
||
assert len(result.history_items) >= 1
|
||
item = result.history_items[0]
|
||
assert item.start_price == 5_000_000
|
||
assert item.start_price_per_m2 == 100_000
|
||
|
||
|
||
def test_validate_match_full_match():
|
||
from scraper_kit.providers.yandex.valuation import ValuationHouseMeta
|
||
|
||
meta = ValuationHouseMeta(year_built=2025, total_floors=25)
|
||
assert meta.validate_match(2025, 25) == 1.0
|
||
|
||
|
||
def test_validate_match_year_mismatch():
|
||
from scraper_kit.providers.yandex.valuation import ValuationHouseMeta
|
||
|
||
meta = ValuationHouseMeta(year_built=1984, total_floors=9)
|
||
assert meta.validate_match(2025, 25) == 0.0
|
||
|
||
|
||
def test_validate_match_partial():
|
||
from scraper_kit.providers.yandex.valuation import ValuationHouseMeta
|
||
|
||
meta = ValuationHouseMeta(year_built=2025, total_floors=10)
|
||
# year matches, floors mismatch (>1 tolerance) → 0.5
|
||
assert meta.validate_match(2025, 25) == 0.5
|
||
|
||
|
||
def test_validate_match_tolerance_one_year():
|
||
from scraper_kit.providers.yandex.valuation import ValuationHouseMeta
|
||
|
||
meta = ValuationHouseMeta(year_built=2024, total_floors=25)
|
||
# ±1 year tolerance
|
||
assert meta.validate_match(2025, 25) == 1.0
|
||
|
||
|
||
def test_validate_match_no_expected_returns_one():
|
||
from scraper_kit.providers.yandex.valuation import ValuationHouseMeta
|
||
|
||
meta = ValuationHouseMeta(year_built=None, total_floors=None)
|
||
assert meta.validate_match(None, None) == 1.0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Sanity cap regression — area_m2 overflow (2026-05-24)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_item_drops_overflow_area():
|
||
"""Regression: area_m2 > 10_000 (e.g. concatenated digits) must be set None,
|
||
not blow up the save batch. Dates and exposure_days should survive.
|
||
|
||
The live bug: chunk-text fallback concatenated digits across a DOM boundary,
|
||
producing area_m2=2_025_106.7 → psycopg.NumericValueOutOfRange on INSERT.
|
||
The cap drops area_m2 to None so the row still saves with area NULL.
|
||
"""
|
||
# Bare concatenated digits (no whitespace between groups) — reproduction of
|
||
# what the chunked-text path emitted in production. Price is placed after
|
||
# the area token so the item stays valid (area_m2 dropped → start_price keeps it).
|
||
text = (
|
||
"3-комнатная 2025106,7 м² 25 этаж "
|
||
"01.07.2024 5 500 000 ₽ В экспозиции 514 дней 26.11.2025"
|
||
)
|
||
item = YandexValuationScraper._parse_item_text(text)
|
||
assert item is not None
|
||
assert item.area_m2 is None, f"expected None for absurd area, got {item.area_m2}"
|
||
assert item.publish_date == date(2024, 7, 1)
|
||
assert item.removed_date == date(2025, 11, 26)
|
||
|
||
|
||
def test_parse_item_keeps_normal_area():
|
||
"""Normal residential area must pass through unchanged."""
|
||
text = (
|
||
"2-комнатная 58,2 м² 3 этаж "
|
||
"10.05.2024 9 300 000 ₽ 159 932 ₽ за м² "
|
||
"9 300 000 ₽ 159 932 ₽ за м² "
|
||
"В экспозиции 30 дней В продаже"
|
||
)
|
||
item = YandexValuationScraper._parse_item_text(text)
|
||
assert item is not None
|
||
assert item.area_m2 == 58.2
|
||
|
||
|
||
def test_parse_item_drops_zero_area():
|
||
text = "1-комнатная 0 м² 3 этаж 10.05.2024 5 000 000 ₽ В продаже"
|
||
item = YandexValuationScraper._parse_item_text(text)
|
||
assert item is not None
|
||
assert item.area_m2 is None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Regex tightening + chronological date ordering (2026-05-24)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_area_regex_rejects_year_concat():
|
||
"""RE_ITEM_AREA must NOT match digits glued onto a preceding number.
|
||
Real bug on prod: 'опубликовано 01.07.2024' + '52,2 м²' rendered into chunked
|
||
text as '202452,2 м²' → matched as 202452.2.
|
||
"""
|
||
text = "2-комнатная 202452,2 м² 3 этаж 01.07.2024 9 300 000 ₽ В продаже"
|
||
item = YandexValuationScraper._parse_item_text(text)
|
||
# The "202452,2" token has digits jammed before it (no separator), so the
|
||
# tightened regex must NOT match it.
|
||
assert (
|
||
item is None or item.area_m2 is None
|
||
), f"expected no area match for concat token, got {item.area_m2 if item else 'None item'}"
|
||
|
||
|
||
def test_area_regex_accepts_isolated_token():
|
||
"""Normal '58,2 м²' surrounded by whitespace must still be parsed."""
|
||
text = "2-комн 58,2 м² 3 этаж 10.05.2024 9 300 000 ₽ В продаже"
|
||
item = YandexValuationScraper._parse_item_text(text)
|
||
assert item is not None
|
||
assert item.area_m2 == 58.2
|
||
|
||
|
||
def test_area_regex_capped_at_4_digits():
|
||
"""Anything > 9999.99 m² must not match (real flats are <500 m²)."""
|
||
text = "квартира 12345 м² 3 этаж 10.05.2024 5 000 000 ₽ В продаже"
|
||
item = YandexValuationScraper._parse_item_text(text)
|
||
# 5-digit area shouldn't match; remaining fields still salvageable
|
||
assert item is None or item.area_m2 is None
|
||
|
||
|
||
def test_dates_sorted_publish_before_removed():
|
||
"""Even if page-text emits removed-date first, parser must put earlier
|
||
date in publish_date and later in removed_date.
|
||
"""
|
||
# Reversed order: removed appears in text before publish
|
||
text = (
|
||
"2-комн 50,0 м² 3 этаж "
|
||
"13.08.2025 В экспозиции 156 дней 12.02.2026 "
|
||
"8 000 000 ₽ В продаже"
|
||
)
|
||
item = YandexValuationScraper._parse_item_text(text)
|
||
assert item is not None
|
||
from datetime import date as _d
|
||
|
||
assert item.publish_date == _d(2025, 8, 13)
|
||
assert item.removed_date == _d(2026, 2, 12)
|
||
|
||
|
||
def test_single_date_is_publish_only():
|
||
"""One date → publish_date set, removed_date None."""
|
||
text = "1-комн 40,0 м² 3 этаж 15.03.2024 В экспозиции 30 дней В продаже"
|
||
item = YandexValuationScraper._parse_item_text(text)
|
||
assert item is not None
|
||
from datetime import date as _d
|
||
|
||
assert item.publish_date == _d(2024, 3, 15)
|
||
assert item.removed_date is None
|
||
|
||
|
||
def test_area_regex_two_komn_chunk():
|
||
"""2-комн chunks were 100% rejected by PR #541 lookbehind. Real bug from
|
||
Базовый 52 sweep: 'rooms=2 → area_m2 is None' for every 2-комн row.
|
||
"""
|
||
# Approximate the chunked text Yandex emits for a 2-room item
|
||
text = "8 000 000 ₽ за м²2-комнатная 52,2 м² 5 этаж 10.05.2024 8 000 000 ₽ В продаже"
|
||
item = YandexValuationScraper._parse_item_text(text)
|
||
assert item is not None
|
||
assert (
|
||
item.area_m2 == 52.2
|
||
), f"2-комн area must parse despite preceding tokens, got {item.area_m2}"
|
||
|
||
|
||
def test_area_regex_still_blocks_year_concat():
|
||
"""The relaxed regex must still reject year+area concat."""
|
||
# '2024' directly fused to '42,6 м²' as can happen after DOM strip
|
||
text = "квартира 202442,6 м² 3 этаж 10.05.2024 9 000 000 ₽ В продаже"
|
||
item = YandexValuationScraper._parse_item_text(text)
|
||
# The '202442,6' token: a `\\d{2,4}` greedy match starting at first '2'
|
||
# gives '2024', then needs '[.,]\\d{1,2}' but '4' follows → no match.
|
||
# Any internal start is blocked by `(?<!\\d)`. So no area parsed.
|
||
assert item is None or item.area_m2 is None
|
||
|
||
|
||
def test_area_regex_still_blocks_subfragment():
|
||
"""Min 2 digits blocks '2,2' fragment from inside '52,2'."""
|
||
# Pure single-digit start without preceding non-digit context
|
||
text = "квартира ,2,2 м² 10.05.2024 5 000 000 ₽ В продаже"
|
||
item = YandexValuationScraper._parse_item_text(text)
|
||
# '2,2' would need (?<!\\d) — `,` is not digit, so lookbehind allows start.
|
||
# But `\\d{2,4}` needs ≥2 leading digits — '2,' only has one. No match.
|
||
assert item is None or item.area_m2 is None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# DOM-based parser (PR refactor — replaces chunked-text fallback)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_DOM_FIXTURE_ROW_HTML = """
|
||
<div class="OffersArchiveSearchOffers__body">
|
||
<div class="OffersArchiveSearchOffers__row">
|
||
<div class="OffersArchiveSearchOffers__cell"></div>
|
||
<div class="OffersArchiveSearchOffers__cell">48,5 м², 1-комнатная19 этаж</div>
|
||
<div class="OffersArchiveSearchOffers__cell">5,1 млн ₽105 155 ₽ за м²</div>
|
||
<div class="OffersArchiveSearchOffers__cell">5,1 млн ₽105 155 ₽ за м²</div>
|
||
<div class="OffersArchiveSearchOffers__cell">23.10.2023В экспозиции 945 дней</div>
|
||
<div class="OffersArchiveSearchOffers__cell">В продаже</div>
|
||
</div>
|
||
<div class="OffersArchiveSearchOffers__row">
|
||
<div class="OffersArchiveSearchOffers__cell"></div>
|
||
<div class="OffersArchiveSearchOffers__cell">58,2 м², 2-комнатная3 этаж</div>
|
||
<div class="OffersArchiveSearchOffers__cell">9,3 млн ₽159 932 ₽ за м²</div>
|
||
<div class="OffersArchiveSearchOffers__cell">8,9 млн ₽152 062 ₽ за м²</div>
|
||
<div class="OffersArchiveSearchOffers__cell">12.02.2026В экспозиции 156 дней</div>
|
||
<div class="OffersArchiveSearchOffers__cell">17.07.2026</div>
|
||
</div>
|
||
</div>
|
||
"""
|
||
|
||
|
||
def test_dom_parser_extracts_two_rows():
|
||
scraper = YandexValuationScraper(config=_KIT_CONFIG)
|
||
house_meta = "Дом 2020 года 25 этажей Монолитное здание "
|
||
result = scraper.parse(
|
||
f"<html><body>{house_meta}{_DOM_FIXTURE_ROW_HTML}</body></html>",
|
||
address="test",
|
||
offer_category="APARTMENT",
|
||
offer_type="SELL",
|
||
page=1,
|
||
source_url="http://test/",
|
||
)
|
||
assert len(result.history_items) == 2
|
||
|
||
# Row 1 — 1-комн in sale
|
||
r1 = result.history_items[0]
|
||
assert r1.area_m2 == 48.5
|
||
assert r1.rooms == 1
|
||
assert r1.floor == 19
|
||
assert r1.start_price == 5_100_000
|
||
assert r1.start_price_per_m2 == 105_155
|
||
assert r1.last_price == 5_100_000
|
||
assert r1.publish_date == date(2023, 10, 23)
|
||
assert r1.exposure_days == 945
|
||
assert r1.removed_date is None
|
||
assert r1.status == "В продаже"
|
||
|
||
# Row 2 — 2-комн sold with both dates
|
||
r2 = result.history_items[1]
|
||
assert r2.area_m2 == 58.2
|
||
assert r2.rooms == 2
|
||
assert r2.floor == 3
|
||
assert r2.start_price == 9_300_000
|
||
assert r2.last_price == 8_900_000
|
||
assert r2.publish_date == date(2026, 2, 12)
|
||
assert r2.removed_date == date(2026, 7, 17)
|
||
assert r2.exposure_days == 156
|
||
|
||
|
||
def test_dom_parser_studio_row():
|
||
html = (
|
||
"<html><body>"
|
||
'<div class="OffersArchiveSearchOffers__row">'
|
||
' <div class="OffersArchiveSearchOffers__cell"></div>'
|
||
' <div class="OffersArchiveSearchOffers__cell">Студия 25 м²5 этаж</div>'
|
||
' <div class="OffersArchiveSearchOffers__cell">4 млн ₽160 000 ₽ за м²</div>'
|
||
' <div class="OffersArchiveSearchOffers__cell">4 млн ₽160 000 ₽ за м²</div>'
|
||
' <div class="OffersArchiveSearchOffers__cell">01.03.2024В экспозиции 30 дней</div>'
|
||
' <div class="OffersArchiveSearchOffers__cell">В продаже</div>'
|
||
"</div>"
|
||
"</body></html>"
|
||
)
|
||
scraper = YandexValuationScraper(config=_KIT_CONFIG)
|
||
result = scraper.parse(
|
||
html,
|
||
address="t",
|
||
offer_category="APARTMENT",
|
||
offer_type="SELL",
|
||
page=1,
|
||
source_url="http://test/",
|
||
)
|
||
assert len(result.history_items) == 1
|
||
r = result.history_items[0]
|
||
assert r.area_m2 == 25.0
|
||
assert r.rooms == 0 # studio
|
||
|
||
|
||
def test_dom_parser_fallback_when_no_rows():
|
||
"""If .OffersArchiveSearchOffers__row matches nothing, fall back to chunked text."""
|
||
# No rows — but body_text has a single item with all info
|
||
html = (
|
||
"<html><body>"
|
||
"Дом 2020 года 16 этажей "
|
||
"2-комнатная 58,0 м² 3 этаж "
|
||
"10.05.2024 9 000 000 ₽ В продаже"
|
||
"</body></html>"
|
||
)
|
||
scraper = YandexValuationScraper(config=_KIT_CONFIG)
|
||
result = scraper.parse(
|
||
html,
|
||
address="t",
|
||
offer_category="APARTMENT",
|
||
offer_type="SELL",
|
||
page=1,
|
||
source_url="http://test/",
|
||
)
|
||
# Fallback path runs — should still find at least price
|
||
# (Don't over-assert: chunked fallback is imperfect; we just want NOT to crash)
|
||
assert isinstance(result.history_items, list)
|
||
|
||
|
||
def test_dom_parser_skips_malformed_row():
|
||
"""Row with <5 cells is skipped (defensive against Yandex A/B variants)."""
|
||
html = (
|
||
"<html><body>"
|
||
'<div class="OffersArchiveSearchOffers__row">'
|
||
' <div class="OffersArchiveSearchOffers__cell">48,5 м²</div>'
|
||
' <div class="OffersArchiveSearchOffers__cell">19 этаж</div>'
|
||
"</div>"
|
||
"</body></html>"
|
||
)
|
||
scraper = YandexValuationScraper(config=_KIT_CONFIG)
|
||
result = scraper.parse(
|
||
html,
|
||
address="t",
|
||
offer_category="APARTMENT",
|
||
offer_type="SELL",
|
||
page=1,
|
||
source_url="http://test/",
|
||
)
|
||
# 2-cell row rejected by _parse_row_cells; fallback also doesn't find a date → 0 items
|
||
assert len(result.history_items) == 0
|