153 lines
5.9 KiB
Python
153 lines
5.9 KiB
Python
"""Tests for YandexDetailScraper structural-source extraction (#1792).
|
|
|
|
Area / ceiling / kitchen / living / floor are extracted from the embedded
|
|
``window.INITIAL_STATE → offerCard.card`` object rather than the h1 title.
|
|
The title misses non-standard layouts (студия / свободная планировка) and
|
|
never carries ceiling/kitchen — which is why DB area coverage sat at ~63%.
|
|
|
|
Fixtures are trimmed real offer pages fetched via the headless browser:
|
|
- yandex_offer_3402418396407468801.html — 1-комн. вторичка (full chars)
|
|
- yandex_offer_7811691822126445498.html — студия / новостройка
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from app.services.scrapers.yandex_detail import (
|
|
YandexDetailScraper,
|
|
_extract_offer_card,
|
|
_parse_card_fields,
|
|
)
|
|
|
|
FIXTURES = Path(__file__).parent / "fixtures"
|
|
SCRAPER = YandexDetailScraper()
|
|
|
|
# offer_id -> expected structural values
|
|
_SECONDARY_ID = "3402418396407468801"
|
|
_STUDIO_ID = "7811691822126445498"
|
|
|
|
|
|
def _load(offer_id: str) -> str:
|
|
return (FIXTURES / f"yandex_offer_{offer_id}.html").read_text(encoding="utf-8")
|
|
|
|
|
|
def _url(offer_id: str) -> str:
|
|
return f"https://realty.yandex.ru/offer/{offer_id}/"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Secondary (1-комнатная) — все структурные поля присутствуют
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestSecondaryOfferStructural:
|
|
def test_area_from_structural_source(self) -> None:
|
|
r = SCRAPER.parse(_load(_SECONDARY_ID), offer_url=_url(_SECONDARY_ID))
|
|
assert r is not None
|
|
assert r.area_m2 == pytest.approx(35.5)
|
|
|
|
def test_rooms_floor_from_structural_source(self) -> None:
|
|
r = SCRAPER.parse(_load(_SECONDARY_ID), offer_url=_url(_SECONDARY_ID))
|
|
assert r is not None
|
|
assert r.rooms == 1
|
|
# floor from floorsOffered[0]; total from floorsTotal
|
|
assert r.floor == 4
|
|
assert r.total_floors == 9
|
|
|
|
def test_ceiling_kitchen_living_present(self) -> None:
|
|
r = SCRAPER.parse(_load(_SECONDARY_ID), offer_url=_url(_SECONDARY_ID))
|
|
assert r is not None
|
|
# These never come from the title — only the structural block carries them.
|
|
assert r.ceiling_height == pytest.approx(2.55)
|
|
assert r.kitchen_area_m2 == pytest.approx(9.4)
|
|
assert r.living_area_m2 == pytest.approx(16.9)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Studio / новостройка — area present even when title says «квартира-студия»
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestStudioOfferStructural:
|
|
def test_area_extracted_for_studio(self) -> None:
|
|
r = SCRAPER.parse(_load(_STUDIO_ID), offer_url=_url(_STUDIO_ID))
|
|
assert r is not None
|
|
# The key regression: studio area must NOT be None.
|
|
assert r.area_m2 is not None
|
|
assert r.area_m2 == pytest.approx(24.92)
|
|
|
|
def test_studio_rooms_zero_from_house_flag(self) -> None:
|
|
r = SCRAPER.parse(_load(_STUDIO_ID), offer_url=_url(_STUDIO_ID))
|
|
assert r is not None
|
|
# roomsTotal is null for studios; house.studio == true → rooms = 0.
|
|
assert r.rooms == 0
|
|
|
|
def test_studio_floor_extracted(self) -> None:
|
|
r = SCRAPER.parse(_load(_STUDIO_ID), offer_url=_url(_STUDIO_ID))
|
|
assert r is not None
|
|
assert r.floor == 9
|
|
assert r.total_floors == 25
|
|
|
|
def test_studio_ceiling_kitchen_legitimately_absent(self) -> None:
|
|
# This is a newbuilding offer — Yandex does not expose ceiling/kitchen/
|
|
# living for it. None here is legitimate, not a parse miss.
|
|
r = SCRAPER.parse(_load(_STUDIO_ID), offer_url=_url(_STUDIO_ID))
|
|
assert r is not None
|
|
assert r.ceiling_height is None
|
|
assert r.kitchen_area_m2 is None
|
|
assert r.living_area_m2 is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helper-level coverage
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestStructuralHelpers:
|
|
def test_extract_offer_card_matches_offer_id(self) -> None:
|
|
card = _extract_offer_card(_load(_SECONDARY_ID), _SECONDARY_ID)
|
|
assert card is not None
|
|
assert card["offerId"] == _SECONDARY_ID
|
|
|
|
def test_extract_offer_card_missing_state_returns_none(self) -> None:
|
|
assert _extract_offer_card("<html><body>no state</body></html>", "123") is None
|
|
|
|
def test_parse_card_fields_space_objects(self) -> None:
|
|
card = {
|
|
"roomsTotal": 2,
|
|
"area": {"value": 55.3, "unit": "SQUARE_METER"},
|
|
"livingSpace": {"value": 30.0, "unit": "SQUARE_METER"},
|
|
"kitchenSpace": {"value": 12.5, "unit": "SQUARE_METER"},
|
|
"ceilingHeight": 2.7,
|
|
"floorsTotal": 18,
|
|
"floorsOffered": [7],
|
|
"house": {"studio": False},
|
|
}
|
|
rooms, area, living, kitchen, ceiling, floor, total = _parse_card_fields(card)
|
|
assert rooms == 2
|
|
assert area == pytest.approx(55.3)
|
|
assert living == pytest.approx(30.0)
|
|
assert kitchen == pytest.approx(12.5)
|
|
assert ceiling == pytest.approx(2.7)
|
|
assert floor == 7
|
|
assert total == 18
|
|
|
|
def test_parse_card_fields_studio_flag_sets_rooms_zero(self) -> None:
|
|
card = {
|
|
"roomsTotal": None,
|
|
"area": {"value": 24.92, "unit": "SQUARE_METER"},
|
|
"floorsOffered": [9],
|
|
"floorsTotal": 25,
|
|
"house": {"studio": True},
|
|
}
|
|
rooms, area, living, kitchen, ceiling, floor, total = _parse_card_fields(card)
|
|
assert rooms == 0
|
|
assert area == pytest.approx(24.92)
|
|
assert living is None
|
|
assert kitchen is None
|
|
assert ceiling is None
|
|
assert floor == 9
|
|
assert total == 25
|