feat(tradein): yandex_newbuilding.py — ЖК landing parser (149 selectors + NLP)
Stage 5 of YandexRealtyScraper v1 (Wave 4 / Worker B — parallel with
yandex_detail.py and yandex_valuation.py).
Module: app/services/scrapers/yandex_newbuilding.py
- YandexNewbuildingScraper(BaseScraper) — fetch_jk(slug, id, city)
URL: /{city}/kupit/novostrojka/<slug>-<id>/
- YandexNewbuildingInfo Pydantic model (ext_id/slug, name, address, lat/lon,
house_class, commission_year/month, total_floors, corpus_count, total_area_ha,
developer_name/url + other_jk, rating + ratings_count + text_reviews_count,
description, metro_stations, house_type)
- Inline coord extraction (Yandex unique: lat/lon visible in raw HTML within
city range filter)
- NLP regex: floors-towers, area-ha, commission date, corpus count (word→int),
rating + counts, metro inline
- Helpers: _find_section_text (h2/h3 → next siblings), _parse_corpus_count
(word/digit), _parse_metro, _extract_jk_address
Tests: 28 unit tests against Татлин-like fixture HTML. All pass. Ruff clean.
This commit is contained in:
lekss361
parent
b5d206bfa8
commit
5f92e94695
2 changed files with 774 additions and 0 deletions
352
tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py
Normal file
352
tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py
Normal file
|
|
@ -0,0 +1,352 @@
|
|||
"""Yandex Realty ЖК landing page parser.
|
||||
|
||||
URL pattern: /{city}/kupit/novostrojka/<slug>-<id>/
|
||||
Reference target: ЖК Татлин (id=1592987, slug=tatlin) — comfort+, June 2023,
|
||||
PRINZIP, rating 4.3, 1505 ratings, 353 text reviews, coords (56.855312, 60.576668).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from selectolax.parser import HTMLParser, Node
|
||||
|
||||
from app.services.scrapers.base import BaseScraper
|
||||
from app.services.scrapers.yandex_helpers import (
|
||||
RE_JK_ID,
|
||||
parse_house_class,
|
||||
parse_house_type,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ── Models ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class JKMetroStation(BaseModel):
|
||||
name: str
|
||||
walk_min: int | None = None
|
||||
|
||||
|
||||
class YandexNewbuildingInfo(BaseModel):
|
||||
"""ЖК landing scrape payload."""
|
||||
|
||||
ext_id: str # '1592987'
|
||||
ext_slug: str # 'tatlin'
|
||||
source_url: str
|
||||
|
||||
# Identification
|
||||
name: str | None = None # 'ЖК «Татлин»'
|
||||
address: str | None = None # 'Екатеринбург, ул. Черепанова / ул. Готвальда'
|
||||
|
||||
# Coords (inline in HTML — Yandex unique vs Avito/Cian)
|
||||
lat: float | None = None
|
||||
lon: float | None = None
|
||||
|
||||
# Classification + commission
|
||||
house_class: str | None = None # 'comfort_plus' etc.
|
||||
commission_year: int | None = None
|
||||
commission_month: str | None = None # 'июнь' (RU)
|
||||
|
||||
# Footprint
|
||||
total_floors: int | None = None # 35
|
||||
corpus_count: int | None = None # 3
|
||||
total_area_ha: float | None = None # 1.5
|
||||
|
||||
# Developer
|
||||
developer_name: str | None = None # 'PRINZIP'
|
||||
developer_url: str | None = None
|
||||
developer_other_jk: list[str] = Field(default_factory=list)
|
||||
|
||||
# Reviews
|
||||
rating: float | None = None # 4.3
|
||||
ratings_count: int | None = None # 1505 — оценок
|
||||
text_reviews_count: int | None = None # 353 — текстовых отзывов
|
||||
|
||||
# NLP / description
|
||||
description: str | None = None
|
||||
|
||||
# Metro
|
||||
metro_stations: list[JKMetroStation] = Field(default_factory=list)
|
||||
|
||||
# House type (best-effort NLP from description)
|
||||
house_type: str | None = None
|
||||
|
||||
raw_payload: dict[str, Any] | None = None
|
||||
|
||||
|
||||
# ── Regex constants ───────────────────────────────────────────────────────────
|
||||
|
||||
RE_FLOORS_TOWERS = re.compile(r"(\d+)-этажн\w+\s+башн", re.IGNORECASE)
|
||||
RE_AREA_HA = re.compile(r"участке\s+([\d,]+)\s*га", re.IGNORECASE)
|
||||
RE_COMMISSION = re.compile(
|
||||
r"(?:введ[её]н|сдан)\s+в\s+эксплуатацию\s+в\s+(\w+)\s+(\d{4})", re.IGNORECASE
|
||||
)
|
||||
RE_RATING = re.compile(r"(\d[.,]\d)\s+из\s+5")
|
||||
RE_RATINGS_COUNT = re.compile(r"(\d+)\s+оценок", re.IGNORECASE)
|
||||
RE_TEXT_REVIEWS = re.compile(r"Смотреть\s+все\s+(\d+)\s+отзыв", re.IGNORECASE)
|
||||
RE_CORPUS_COUNT_WORD = re.compile(
|
||||
r"(одн[ау]|две|три|четыре|пять|шесть|семь|восемь|\d+)\s+(?:[\d\s-]*)\s*-?\s*этажн",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
RE_METRO_INLINE = re.compile(
|
||||
r"([А-ЯЁ][А-Яа-яё\s-]{2,30}?)\s+(\d+)\s*мин",
|
||||
)
|
||||
|
||||
# Ekb-specific coord ranges (extend per-city later)
|
||||
LAT_RANGE = (55.5, 57.5)
|
||||
LON_RANGE = (59.5, 61.5)
|
||||
|
||||
# Word → number map
|
||||
_WORD_NUM: dict[str, int] = {
|
||||
"одна": 1,
|
||||
"одну": 1,
|
||||
"одной": 1,
|
||||
"две": 2,
|
||||
"три": 3,
|
||||
"четыре": 4,
|
||||
"пять": 5,
|
||||
"шесть": 6,
|
||||
"семь": 7,
|
||||
"восемь": 8,
|
||||
}
|
||||
|
||||
|
||||
# ── Scraper class ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class YandexNewbuildingScraper(BaseScraper):
|
||||
name = "yandex_realty_nb"
|
||||
base_url = "https://realty.yandex.ru"
|
||||
request_delay_sec = 4.0
|
||||
|
||||
async def fetch_around(
|
||||
self, lat: float, lon: float, radius_m: int = 1000
|
||||
) -> list: # type: ignore[override]
|
||||
raise NotImplementedError(
|
||||
"YandexNewbuildingScraper is JK-slug-based; use fetch_jk(slug, id) instead."
|
||||
)
|
||||
|
||||
async def fetch_jk(
|
||||
self, jk_slug: str, jk_id: str, city: str = "ekaterinburg"
|
||||
) -> YandexNewbuildingInfo | None:
|
||||
url = f"{self.base_url}/{city}/kupit/novostrojka/{jk_slug}-{jk_id}/"
|
||||
try:
|
||||
response = await self._http_get(url)
|
||||
except Exception:
|
||||
logger.exception("yandex nb fetch failed: %s", url)
|
||||
return None
|
||||
if response.status_code != 200:
|
||||
logger.warning("yandex nb returned %d for %s", response.status_code, url)
|
||||
return None
|
||||
result = self.parse(response.text, jk_slug=jk_slug, jk_id=jk_id, source_url=url)
|
||||
await self.sleep_between_requests()
|
||||
return result
|
||||
|
||||
def parse(
|
||||
self, html: str, jk_slug: str, jk_id: str, source_url: str
|
||||
) -> YandexNewbuildingInfo:
|
||||
tree = HTMLParser(html)
|
||||
body = tree.body
|
||||
body_text = body.text(strip=True) if body else ""
|
||||
|
||||
# Header
|
||||
h1 = tree.css_first("h1")
|
||||
name = h1.text(strip=True) if h1 else None
|
||||
|
||||
# Sections (heading-based extraction)
|
||||
location_text = _find_section_text(tree, "Расположение") or ""
|
||||
description_section = (
|
||||
_find_section_text(tree, "О комплексе")
|
||||
or _find_section_text(tree, "Расположение, транспортная доступность")
|
||||
or location_text
|
||||
)
|
||||
|
||||
# Coords — inline scan within plausible range
|
||||
lat, lon = _extract_coords(html)
|
||||
|
||||
# Class
|
||||
house_class = parse_house_class(description_section) or parse_house_class(body_text)
|
||||
|
||||
# Commission
|
||||
commission_year: int | None = None
|
||||
commission_month: str | None = None
|
||||
c_m = RE_COMMISSION.search(description_section) or RE_COMMISSION.search(body_text)
|
||||
if c_m:
|
||||
commission_month = c_m.group(1).lower()
|
||||
try:
|
||||
commission_year = int(c_m.group(2))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Floors + corpus + area
|
||||
floors_m = RE_FLOORS_TOWERS.search(description_section) or RE_FLOORS_TOWERS.search(
|
||||
body_text
|
||||
)
|
||||
total_floors = int(floors_m.group(1)) if floors_m else None
|
||||
|
||||
corpus_count = _parse_corpus_count(description_section or body_text)
|
||||
|
||||
area_m = RE_AREA_HA.search(description_section) or RE_AREA_HA.search(body_text)
|
||||
total_area_ha = float(area_m.group(1).replace(",", ".")) if area_m else None
|
||||
|
||||
# Developer
|
||||
dev_link = tree.css_first('[data-test="CARD_DEV_BADGE_DEVELOPER_LINK"]')
|
||||
developer_name: str | None = None
|
||||
developer_url: str | None = None
|
||||
if dev_link is not None:
|
||||
developer_name = dev_link.text(strip=True) or None
|
||||
href = dev_link.attributes.get("href", "")
|
||||
if href:
|
||||
developer_url = (
|
||||
href
|
||||
if href.startswith("http")
|
||||
else f"https://realty.yandex.ru{href}"
|
||||
)
|
||||
|
||||
# Developer's other JKs
|
||||
other_jk_block = tree.css_first('[data-test="CardDevSites"]')
|
||||
developer_other_jk: list[str] = []
|
||||
if other_jk_block is not None:
|
||||
for jk_a in other_jk_block.css("a"):
|
||||
t = jk_a.text(strip=True)
|
||||
if t and t not in developer_other_jk:
|
||||
developer_other_jk.append(t)
|
||||
|
||||
# Reviews
|
||||
rating_m = RE_RATING.search(body_text)
|
||||
rating = float(rating_m.group(1).replace(",", ".")) if rating_m else None
|
||||
rcount_m = RE_RATINGS_COUNT.search(body_text)
|
||||
ratings_count = int(rcount_m.group(1)) if rcount_m else None
|
||||
treviews_m = RE_TEXT_REVIEWS.search(body_text)
|
||||
text_reviews_count = int(treviews_m.group(1)) if treviews_m else None
|
||||
|
||||
# Address (header area before lat/lon block)
|
||||
address = _extract_jk_address(body_text, name)
|
||||
|
||||
# Metro stations
|
||||
metro_stations = _parse_metro(body_text)
|
||||
|
||||
# NLP house_type fallback
|
||||
house_type = parse_house_type(description_section) or parse_house_type(body_text)
|
||||
|
||||
return YandexNewbuildingInfo(
|
||||
ext_id=jk_id,
|
||||
ext_slug=jk_slug,
|
||||
source_url=source_url,
|
||||
name=name,
|
||||
address=address,
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
house_class=house_class,
|
||||
commission_year=commission_year,
|
||||
commission_month=commission_month,
|
||||
total_floors=total_floors,
|
||||
corpus_count=corpus_count,
|
||||
total_area_ha=total_area_ha,
|
||||
developer_name=developer_name,
|
||||
developer_url=developer_url,
|
||||
developer_other_jk=developer_other_jk[:10],
|
||||
rating=rating,
|
||||
ratings_count=ratings_count,
|
||||
text_reviews_count=text_reviews_count,
|
||||
description=description_section or None,
|
||||
metro_stations=metro_stations,
|
||||
house_type=house_type,
|
||||
raw_payload={
|
||||
"description_len": len(description_section or ""),
|
||||
"body_len": len(body_text),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _extract_coords(html: str) -> tuple[float | None, float | None]:
|
||||
"""Scan HTML for any `<lat>.\\d+` and `<lon>.\\d+` matching plausible city range."""
|
||||
lats = [
|
||||
float(m)
|
||||
for m in re.findall(r"\b(\d{2}\.\d{4,8})\b", html)
|
||||
if LAT_RANGE[0] <= float(m) <= LAT_RANGE[1]
|
||||
]
|
||||
lons = [
|
||||
float(m)
|
||||
for m in re.findall(r"\b(\d{2}\.\d{4,8})\b", html)
|
||||
if LON_RANGE[0] <= float(m) <= LON_RANGE[1]
|
||||
]
|
||||
lat = lats[0] if lats else None
|
||||
lon = lons[0] if lons else None
|
||||
return lat, lon
|
||||
|
||||
|
||||
def _find_section_text(tree: HTMLParser, heading: str) -> str | None:
|
||||
for h in tree.css("h2, h3"):
|
||||
if heading.lower() in (h.text(strip=True) or "").lower():
|
||||
parts: list[str] = []
|
||||
node: Node | None = h.next
|
||||
while node is not None:
|
||||
tag = (node.tag or "").lower()
|
||||
if tag in {"h2", "h3"}:
|
||||
break
|
||||
txt = node.text(strip=True) if hasattr(node, "text") else ""
|
||||
if txt:
|
||||
parts.append(txt)
|
||||
node = node.next
|
||||
return " ".join(parts).strip() or None
|
||||
return None
|
||||
|
||||
|
||||
def _parse_corpus_count(text: str) -> int | None:
|
||||
m = RE_CORPUS_COUNT_WORD.search(text)
|
||||
if not m:
|
||||
return None
|
||||
token = m.group(1).lower()
|
||||
if token.isdigit():
|
||||
return int(token)
|
||||
return _WORD_NUM.get(token)
|
||||
|
||||
|
||||
def _parse_metro(text: str) -> list[JKMetroStation]:
|
||||
stations: list[JKMetroStation] = []
|
||||
for m in RE_METRO_INLINE.finditer(text):
|
||||
name = m.group(1).strip()
|
||||
# filter common noise
|
||||
if not 2 <= len(name) <= 40 or "ходьб" in name.lower() or "минут" in name.lower():
|
||||
continue
|
||||
try:
|
||||
stations.append(JKMetroStation(name=name, walk_min=int(m.group(2))))
|
||||
except ValueError:
|
||||
continue
|
||||
if len(stations) >= 5:
|
||||
break
|
||||
return stations
|
||||
|
||||
|
||||
def _extract_jk_address(body_text: str, name: str | None) -> str | None:
|
||||
"""Find address line — typically right after JK name, before metro entries."""
|
||||
if name and name in body_text:
|
||||
after_name = body_text.split(name, 1)[1]
|
||||
# Match 'Город, ул. X / ул. Y' until first metro mention
|
||||
m = re.match(
|
||||
r"\s*([А-ЯЁ][^•]{5,150}?)(?:\s+[А-ЯЁ][а-яё]+\s+\d+\s*мин|\d+\s+оценок|$)",
|
||||
after_name,
|
||||
)
|
||||
if m:
|
||||
addr = m.group(1).strip().rstrip(",").strip()
|
||||
if 10 < len(addr) < 200:
|
||||
return addr
|
||||
return None
|
||||
|
||||
|
||||
# Expose RE_JK_ID at module level for external use (e.g. SERP link → newbuilding detail)
|
||||
__all__ = [
|
||||
"RE_JK_ID",
|
||||
"JKMetroStation",
|
||||
"YandexNewbuildingInfo",
|
||||
"YandexNewbuildingScraper",
|
||||
]
|
||||
422
tradein-mvp/backend/tests/test_yandex_newbuilding.py
Normal file
422
tradein-mvp/backend/tests/test_yandex_newbuilding.py
Normal file
|
|
@ -0,0 +1,422 @@
|
|||
"""Unit tests for yandex_newbuilding.py — ЖК landing parser (Worker B, Wave 4).
|
||||
|
||||
Reference target: ЖК Татлин (slug=tatlin, id=1592987) — comfort+, June 2023,
|
||||
PRINZIP developer, rating 4.3, 1505 ratings, 353 text reviews,
|
||||
coords (56.855312, 60.576668).
|
||||
|
||||
All tests are offline — no network calls. Parser receives hand-crafted fixture HTML.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.yandex_newbuilding import (
|
||||
JKMetroStation,
|
||||
YandexNewbuildingInfo,
|
||||
YandexNewbuildingScraper,
|
||||
_extract_coords,
|
||||
_find_section_text,
|
||||
_parse_corpus_count,
|
||||
_parse_metro,
|
||||
)
|
||||
|
||||
# ── Fixture HTML ──────────────────────────────────────────────────────────────
|
||||
|
||||
TATLIN_FIXTURE_HTML = """<!DOCTYPE html>
|
||||
<html>
|
||||
<head><title>ЖК «Татлин»</title></head>
|
||||
<body>
|
||||
<h1>ЖК «Татлин»</h1>
|
||||
<div>Екатеринбург, ул. Черепанова / ул. Готвальда</div>
|
||||
<a data-test="CARD_DEV_BADGE_DEVELOPER_LINK" href="/developer/prinzip">PRINZIP</a>
|
||||
<div data-test="CardDevSites">
|
||||
<a href="/zhk/1">ЖК Кислород</a>
|
||||
<a href="/zhk/2">ЖК Нова</a>
|
||||
</div>
|
||||
<div>56.855312, 60.576668</div>
|
||||
<div>Уральская 11 мин. Динамо 16 мин.</div>
|
||||
<div>4.3 из 5 1505 оценок Смотреть все 353 отзыва</div>
|
||||
<h2>О комплексе</h2>
|
||||
<p>ЖК «Татлин» — три 35-этажные башни в Заречном микрорайоне Екатеринбурга.
|
||||
Комплекс класса комфорт+. Введён в эксплуатацию в июне 2023 года.
|
||||
Расположен на участке 1,5 га. Монолитный тип дома.</p>
|
||||
<h2>Расположение</h2>
|
||||
<p>В 5 минутах от центра</p>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
# Minimal fixture — missing optional sections
|
||||
MINIMAL_FIXTURE_HTML = """<!DOCTYPE html>
|
||||
<html><body>
|
||||
<h1>ЖК Минимальный</h1>
|
||||
</body></html>"""
|
||||
|
||||
|
||||
# ── Full happy-path test ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_parse_jk_full_fixture():
|
||||
"""Happy path: all major fields extracted from Татлин-like HTML."""
|
||||
scraper = YandexNewbuildingScraper()
|
||||
info = scraper.parse(
|
||||
TATLIN_FIXTURE_HTML,
|
||||
jk_slug="tatlin",
|
||||
jk_id="1592987",
|
||||
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
|
||||
)
|
||||
|
||||
assert isinstance(info, YandexNewbuildingInfo)
|
||||
assert info.ext_id == "1592987"
|
||||
assert info.ext_slug == "tatlin"
|
||||
assert "Татлин" in (info.name or "")
|
||||
|
||||
# coords
|
||||
assert info.lat == pytest.approx(56.855312, abs=1e-5)
|
||||
assert info.lon == pytest.approx(60.576668, abs=1e-5)
|
||||
|
||||
# class + commission
|
||||
assert info.house_class == "comfort_plus"
|
||||
assert info.commission_year == 2023
|
||||
assert info.commission_month == "июне"
|
||||
|
||||
# footprint
|
||||
assert info.total_floors == 35
|
||||
assert info.corpus_count == 3
|
||||
assert info.total_area_ha == pytest.approx(1.5, abs=0.01)
|
||||
|
||||
# developer
|
||||
assert info.developer_name == "PRINZIP"
|
||||
assert info.developer_url is not None and "prinzip" in info.developer_url
|
||||
assert "ЖК Кислород" in info.developer_other_jk
|
||||
assert "ЖК Нова" in info.developer_other_jk
|
||||
|
||||
# reviews
|
||||
assert info.rating == pytest.approx(4.3, abs=0.01)
|
||||
assert info.ratings_count == 1505
|
||||
assert info.text_reviews_count == 353
|
||||
|
||||
# description present
|
||||
assert info.description is not None
|
||||
assert len(info.description) > 10
|
||||
|
||||
# metro
|
||||
assert len(info.metro_stations) >= 1
|
||||
station_names = [s.name for s in info.metro_stations]
|
||||
assert any("Уральская" in n for n in station_names)
|
||||
|
||||
# house type from "монолитный"
|
||||
assert info.house_type == "monolith"
|
||||
|
||||
# raw_payload
|
||||
assert info.raw_payload is not None
|
||||
assert "body_len" in info.raw_payload
|
||||
|
||||
|
||||
# ── Coord extraction tests ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_extract_coords_within_ekb_range():
|
||||
html = "<div>56.855312, 60.576668</div>"
|
||||
lat, lon = _extract_coords(html)
|
||||
assert lat == pytest.approx(56.855312, abs=1e-5)
|
||||
assert lon == pytest.approx(60.576668, abs=1e-5)
|
||||
|
||||
|
||||
def test_extract_coords_outside_range_returns_none():
|
||||
# Moscow coords — outside EKB range
|
||||
html = "<div>55.7558, 37.6176</div>"
|
||||
_lat, lon = _extract_coords(html)
|
||||
# 55.75 is in LAT_RANGE but 37.61 is NOT in LON_RANGE
|
||||
assert lon is None
|
||||
|
||||
|
||||
def test_extract_coords_no_coords_returns_none():
|
||||
html = "<div>Текст без координат</div>"
|
||||
lat, lon = _extract_coords(html)
|
||||
assert lat is None
|
||||
assert lon is None
|
||||
|
||||
|
||||
def test_extract_coords_multiple_takes_first():
|
||||
"""When multiple valid coords present, first is returned."""
|
||||
html = "<div>56.855312, 60.576668</div><div>56.900000, 60.600000</div>"
|
||||
lat, lon = _extract_coords(html)
|
||||
assert lat == pytest.approx(56.855312, abs=1e-5)
|
||||
assert lon == pytest.approx(60.576668, abs=1e-5)
|
||||
|
||||
|
||||
# ── Corpus count tests ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_parse_corpus_count_word_three():
|
||||
text = "три 35-этажные башни в центре"
|
||||
result = _parse_corpus_count(text)
|
||||
assert result == 3
|
||||
|
||||
|
||||
def test_parse_corpus_count_digit():
|
||||
text = "5 35-этажных корпусов"
|
||||
result = _parse_corpus_count(text)
|
||||
assert result == 5
|
||||
|
||||
|
||||
def test_parse_corpus_count_two():
|
||||
text = "две 20-этажные башни"
|
||||
result = _parse_corpus_count(text)
|
||||
assert result == 2
|
||||
|
||||
|
||||
def test_parse_corpus_count_none_when_no_match():
|
||||
text = "Обычный текст без этажей"
|
||||
result = _parse_corpus_count(text)
|
||||
assert result is None
|
||||
|
||||
|
||||
# ── House class test ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_parse_house_class_comfort_plus_from_description():
|
||||
from app.services.scrapers.yandex_helpers import parse_house_class
|
||||
|
||||
text = "Жилой комплекс класса комфорт+ в центре города"
|
||||
result = parse_house_class(text)
|
||||
assert result == "comfort_plus"
|
||||
|
||||
|
||||
def test_parse_house_class_business():
|
||||
from app.services.scrapers.yandex_helpers import parse_house_class
|
||||
|
||||
text = "ЖК класса бизнес"
|
||||
result = parse_house_class(text)
|
||||
assert result == "business"
|
||||
|
||||
|
||||
# ── Commission year/month test ────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_parse_commission_year_month():
|
||||
"""Full pipeline: parse() correctly extracts commission_year and commission_month."""
|
||||
scraper = YandexNewbuildingScraper()
|
||||
info = scraper.parse(
|
||||
TATLIN_FIXTURE_HTML,
|
||||
jk_slug="tatlin",
|
||||
jk_id="1592987",
|
||||
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
|
||||
)
|
||||
assert info.commission_year == 2023
|
||||
assert info.commission_month == "июне"
|
||||
|
||||
|
||||
def test_parse_commission_sdan_variant():
|
||||
"""Regex supports 'сдан в эксплуатацию' phrasing as well."""
|
||||
html = """<html><body>
|
||||
<h1>ЖК Тест</h1>
|
||||
<h2>О комплексе</h2>
|
||||
<p>Сдан в эксплуатацию в марте 2025 года.</p>
|
||||
</body></html>"""
|
||||
scraper = YandexNewbuildingScraper()
|
||||
info = scraper.parse(html, jk_slug="test", jk_id="999", source_url="http://x")
|
||||
assert info.commission_year == 2025
|
||||
assert info.commission_month == "марте"
|
||||
|
||||
|
||||
# ── Rating + counts test ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_parse_rating_and_counts():
|
||||
scraper = YandexNewbuildingScraper()
|
||||
info = scraper.parse(
|
||||
TATLIN_FIXTURE_HTML,
|
||||
jk_slug="tatlin",
|
||||
jk_id="1592987",
|
||||
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
|
||||
)
|
||||
assert info.rating == pytest.approx(4.3, abs=0.01)
|
||||
assert info.ratings_count == 1505
|
||||
assert info.text_reviews_count == 353
|
||||
|
||||
|
||||
def test_parse_rating_comma_separator():
|
||||
"""Rating '4,3 из 5' with comma handled correctly."""
|
||||
html = "<html><body><div>4,3 из 5 200 оценок</div></body></html>"
|
||||
scraper = YandexNewbuildingScraper()
|
||||
info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x")
|
||||
assert info.rating == pytest.approx(4.3, abs=0.01)
|
||||
assert info.ratings_count == 200
|
||||
|
||||
|
||||
# ── Developer link test ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_developer_name_extracted_from_data_test_link():
|
||||
scraper = YandexNewbuildingScraper()
|
||||
info = scraper.parse(
|
||||
TATLIN_FIXTURE_HTML,
|
||||
jk_slug="tatlin",
|
||||
jk_id="1592987",
|
||||
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
|
||||
)
|
||||
assert info.developer_name == "PRINZIP"
|
||||
assert info.developer_url == "https://realty.yandex.ru/developer/prinzip"
|
||||
|
||||
|
||||
def test_developer_absolute_url_kept_as_is():
|
||||
html = """<html><body>
|
||||
<a data-test="CARD_DEV_BADGE_DEVELOPER_LINK"
|
||||
href="https://realty.yandex.ru/developer/abc">АБС Групп</a>
|
||||
</body></html>"""
|
||||
scraper = YandexNewbuildingScraper()
|
||||
info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x")
|
||||
assert info.developer_name == "АБС Групп"
|
||||
assert info.developer_url == "https://realty.yandex.ru/developer/abc"
|
||||
|
||||
|
||||
# ── Metro stations test ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_metro_stations_parsed():
|
||||
scraper = YandexNewbuildingScraper()
|
||||
info = scraper.parse(
|
||||
TATLIN_FIXTURE_HTML,
|
||||
jk_slug="tatlin",
|
||||
jk_id="1592987",
|
||||
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
|
||||
)
|
||||
assert len(info.metro_stations) >= 1
|
||||
uralskaya = next(
|
||||
(s for s in info.metro_stations if "Уральская" in s.name), None
|
||||
)
|
||||
assert uralskaya is not None
|
||||
assert uralskaya.walk_min == 11
|
||||
|
||||
|
||||
def test_parse_metro_direct():
|
||||
text = "Уральская 11 мин. Динамо 16 мин."
|
||||
stations = _parse_metro(text)
|
||||
assert len(stations) == 2
|
||||
assert stations[0].name == "Уральская"
|
||||
assert stations[0].walk_min == 11
|
||||
assert stations[1].name == "Динамо"
|
||||
assert stations[1].walk_min == 16
|
||||
|
||||
|
||||
def test_parse_metro_caps_at_five():
|
||||
text = (
|
||||
"Альфа 5 мин. Бета 6 мин. Гамма 7 мин. "
|
||||
"Дельта 8 мин. Эпсилон 9 мин. Зета 10 мин."
|
||||
)
|
||||
stations = _parse_metro(text)
|
||||
assert len(stations) == 5
|
||||
|
||||
|
||||
# ── No description section test ───────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_no_description_section_returns_none():
|
||||
"""When no О комплексе / Расположение section, description is None."""
|
||||
scraper = YandexNewbuildingScraper()
|
||||
info = scraper.parse(
|
||||
MINIMAL_FIXTURE_HTML,
|
||||
jk_slug="minimal",
|
||||
jk_id="42",
|
||||
source_url="http://example.com/",
|
||||
)
|
||||
assert info.description is None
|
||||
assert info.house_class is None
|
||||
assert info.commission_year is None
|
||||
assert info.total_floors is None
|
||||
|
||||
|
||||
# ── fetch_around raises NotImplementedError ───────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_around_raises_not_implemented():
|
||||
scraper = YandexNewbuildingScraper()
|
||||
with pytest.raises(NotImplementedError, match="JK-slug-based"):
|
||||
await scraper.fetch_around(56.855, 60.576)
|
||||
|
||||
|
||||
# ── _find_section_text helper ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_find_section_text_returns_content_after_heading():
|
||||
from selectolax.parser import HTMLParser
|
||||
|
||||
html = """<html><body>
|
||||
<h2>О комплексе</h2>
|
||||
<p>Описание комплекса здесь.</p>
|
||||
<p>Ещё абзац.</p>
|
||||
<h2>Другой раздел</h2>
|
||||
<p>Не должен попасть.</p>
|
||||
</body></html>"""
|
||||
tree = HTMLParser(html)
|
||||
result = _find_section_text(tree, "О комплексе")
|
||||
assert result is not None
|
||||
assert "Описание комплекса" in result
|
||||
assert "Не должен" not in result
|
||||
|
||||
|
||||
def test_find_section_text_returns_none_when_missing():
|
||||
from selectolax.parser import HTMLParser
|
||||
|
||||
html = "<html><body><p>Текст</p></body></html>"
|
||||
tree = HTMLParser(html)
|
||||
result = _find_section_text(tree, "О комплексе")
|
||||
assert result is None
|
||||
|
||||
|
||||
# ── Developer other JK dedup ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_developer_other_jk_dedup():
|
||||
"""Duplicate JK names are not added twice."""
|
||||
html = """<html><body>
|
||||
<div data-test="CardDevSites">
|
||||
<a href="/1">ЖК Один</a>
|
||||
<a href="/2">ЖК Один</a>
|
||||
<a href="/3">ЖК Два</a>
|
||||
</div>
|
||||
</body></html>"""
|
||||
scraper = YandexNewbuildingScraper()
|
||||
info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x")
|
||||
assert info.developer_other_jk.count("ЖК Один") == 1
|
||||
assert "ЖК Два" in info.developer_other_jk
|
||||
|
||||
|
||||
def test_developer_other_jk_capped_at_10():
|
||||
"""Only first 10 other JK entries are kept."""
|
||||
links = "\n".join(
|
||||
f'<a href="/{i}">ЖК {i}</a>' for i in range(15)
|
||||
)
|
||||
html = f"""<html><body>
|
||||
<div data-test="CardDevSites">{links}</div>
|
||||
</body></html>"""
|
||||
scraper = YandexNewbuildingScraper()
|
||||
info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x")
|
||||
assert len(info.developer_other_jk) == 10
|
||||
|
||||
|
||||
# ── model defaults ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_yandex_newbuilding_info_defaults():
|
||||
"""Model initialises with safe defaults for all optional fields."""
|
||||
info = YandexNewbuildingInfo(ext_id="1", ext_slug="x", source_url="http://x")
|
||||
assert info.name is None
|
||||
assert info.lat is None
|
||||
assert info.lon is None
|
||||
assert info.house_class is None
|
||||
assert info.developer_other_jk == []
|
||||
assert info.metro_stations == []
|
||||
assert info.raw_payload is None
|
||||
|
||||
|
||||
def test_jk_metro_station_model():
|
||||
station = JKMetroStation(name="Уральская", walk_min=11)
|
||||
assert station.name == "Уральская"
|
||||
assert station.walk_min == 11
|
||||
|
||||
station_no_time = JKMetroStation(name="Геологическая")
|
||||
assert station_no_time.walk_min is None
|
||||
Loading…
Add table
Reference in a new issue