diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py b/tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py new file mode 100644 index 00000000..cdca4db5 --- /dev/null +++ b/tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py @@ -0,0 +1,352 @@ +"""Yandex Realty ЖК landing page parser. + +URL pattern: /{city}/kupit/novostrojka/-/ +Reference target: ЖК Татлин (id=1592987, slug=tatlin) — comfort+, June 2023, +PRINZIP, rating 4.3, 1505 ratings, 353 text reviews, coords (56.855312, 60.576668). +""" + +from __future__ import annotations + +import logging +import re +from typing import Any + +from pydantic import BaseModel, Field +from selectolax.parser import HTMLParser, Node + +from app.services.scrapers.base import BaseScraper +from app.services.scrapers.yandex_helpers import ( + RE_JK_ID, + parse_house_class, + parse_house_type, +) + +logger = logging.getLogger(__name__) + + +# ── Models ─────────────────────────────────────────────────────────────────── + + +class JKMetroStation(BaseModel): + name: str + walk_min: int | None = None + + +class YandexNewbuildingInfo(BaseModel): + """ЖК landing scrape payload.""" + + ext_id: str # '1592987' + ext_slug: str # 'tatlin' + source_url: str + + # Identification + name: str | None = None # 'ЖК «Татлин»' + address: str | None = None # 'Екатеринбург, ул. Черепанова / ул. Готвальда' + + # Coords (inline in HTML — Yandex unique vs Avito/Cian) + lat: float | None = None + lon: float | None = None + + # Classification + commission + house_class: str | None = None # 'comfort_plus' etc. + commission_year: int | None = None + commission_month: str | None = None # 'июнь' (RU) + + # Footprint + total_floors: int | None = None # 35 + corpus_count: int | None = None # 3 + total_area_ha: float | None = None # 1.5 + + # Developer + developer_name: str | None = None # 'PRINZIP' + developer_url: str | None = None + developer_other_jk: list[str] = Field(default_factory=list) + + # Reviews + rating: float | None = None # 4.3 + ratings_count: int | None = None # 1505 — оценок + text_reviews_count: int | None = None # 353 — текстовых отзывов + + # NLP / description + description: str | None = None + + # Metro + metro_stations: list[JKMetroStation] = Field(default_factory=list) + + # House type (best-effort NLP from description) + house_type: str | None = None + + raw_payload: dict[str, Any] | None = None + + +# ── Regex constants ─────────────────────────────────────────────────────────── + +RE_FLOORS_TOWERS = re.compile(r"(\d+)-этажн\w+\s+башн", re.IGNORECASE) +RE_AREA_HA = re.compile(r"участке\s+([\d,]+)\s*га", re.IGNORECASE) +RE_COMMISSION = re.compile( + r"(?:введ[её]н|сдан)\s+в\s+эксплуатацию\s+в\s+(\w+)\s+(\d{4})", re.IGNORECASE +) +RE_RATING = re.compile(r"(\d[.,]\d)\s+из\s+5") +RE_RATINGS_COUNT = re.compile(r"(\d+)\s+оценок", re.IGNORECASE) +RE_TEXT_REVIEWS = re.compile(r"Смотреть\s+все\s+(\d+)\s+отзыв", re.IGNORECASE) +RE_CORPUS_COUNT_WORD = re.compile( + r"(одн[ау]|две|три|четыре|пять|шесть|семь|восемь|\d+)\s+(?:[\d\s-]*)\s*-?\s*этажн", + re.IGNORECASE, +) +RE_METRO_INLINE = re.compile( + r"([А-ЯЁ][А-Яа-яё\s-]{2,30}?)\s+(\d+)\s*мин", +) + +# Ekb-specific coord ranges (extend per-city later) +LAT_RANGE = (55.5, 57.5) +LON_RANGE = (59.5, 61.5) + +# Word → number map +_WORD_NUM: dict[str, int] = { + "одна": 1, + "одну": 1, + "одной": 1, + "две": 2, + "три": 3, + "четыре": 4, + "пять": 5, + "шесть": 6, + "семь": 7, + "восемь": 8, +} + + +# ── Scraper class ───────────────────────────────────────────────────────────── + + +class YandexNewbuildingScraper(BaseScraper): + name = "yandex_realty_nb" + base_url = "https://realty.yandex.ru" + request_delay_sec = 4.0 + + async def fetch_around( + self, lat: float, lon: float, radius_m: int = 1000 + ) -> list: # type: ignore[override] + raise NotImplementedError( + "YandexNewbuildingScraper is JK-slug-based; use fetch_jk(slug, id) instead." + ) + + async def fetch_jk( + self, jk_slug: str, jk_id: str, city: str = "ekaterinburg" + ) -> YandexNewbuildingInfo | None: + url = f"{self.base_url}/{city}/kupit/novostrojka/{jk_slug}-{jk_id}/" + try: + response = await self._http_get(url) + except Exception: + logger.exception("yandex nb fetch failed: %s", url) + return None + if response.status_code != 200: + logger.warning("yandex nb returned %d for %s", response.status_code, url) + return None + result = self.parse(response.text, jk_slug=jk_slug, jk_id=jk_id, source_url=url) + await self.sleep_between_requests() + return result + + def parse( + self, html: str, jk_slug: str, jk_id: str, source_url: str + ) -> YandexNewbuildingInfo: + tree = HTMLParser(html) + body = tree.body + body_text = body.text(strip=True) if body else "" + + # Header + h1 = tree.css_first("h1") + name = h1.text(strip=True) if h1 else None + + # Sections (heading-based extraction) + location_text = _find_section_text(tree, "Расположение") or "" + description_section = ( + _find_section_text(tree, "О комплексе") + or _find_section_text(tree, "Расположение, транспортная доступность") + or location_text + ) + + # Coords — inline scan within plausible range + lat, lon = _extract_coords(html) + + # Class + house_class = parse_house_class(description_section) or parse_house_class(body_text) + + # Commission + commission_year: int | None = None + commission_month: str | None = None + c_m = RE_COMMISSION.search(description_section) or RE_COMMISSION.search(body_text) + if c_m: + commission_month = c_m.group(1).lower() + try: + commission_year = int(c_m.group(2)) + except ValueError: + pass + + # Floors + corpus + area + floors_m = RE_FLOORS_TOWERS.search(description_section) or RE_FLOORS_TOWERS.search( + body_text + ) + total_floors = int(floors_m.group(1)) if floors_m else None + + corpus_count = _parse_corpus_count(description_section or body_text) + + area_m = RE_AREA_HA.search(description_section) or RE_AREA_HA.search(body_text) + total_area_ha = float(area_m.group(1).replace(",", ".")) if area_m else None + + # Developer + dev_link = tree.css_first('[data-test="CARD_DEV_BADGE_DEVELOPER_LINK"]') + developer_name: str | None = None + developer_url: str | None = None + if dev_link is not None: + developer_name = dev_link.text(strip=True) or None + href = dev_link.attributes.get("href", "") + if href: + developer_url = ( + href + if href.startswith("http") + else f"https://realty.yandex.ru{href}" + ) + + # Developer's other JKs + other_jk_block = tree.css_first('[data-test="CardDevSites"]') + developer_other_jk: list[str] = [] + if other_jk_block is not None: + for jk_a in other_jk_block.css("a"): + t = jk_a.text(strip=True) + if t and t not in developer_other_jk: + developer_other_jk.append(t) + + # Reviews + rating_m = RE_RATING.search(body_text) + rating = float(rating_m.group(1).replace(",", ".")) if rating_m else None + rcount_m = RE_RATINGS_COUNT.search(body_text) + ratings_count = int(rcount_m.group(1)) if rcount_m else None + treviews_m = RE_TEXT_REVIEWS.search(body_text) + text_reviews_count = int(treviews_m.group(1)) if treviews_m else None + + # Address (header area before lat/lon block) + address = _extract_jk_address(body_text, name) + + # Metro stations + metro_stations = _parse_metro(body_text) + + # NLP house_type fallback + house_type = parse_house_type(description_section) or parse_house_type(body_text) + + return YandexNewbuildingInfo( + ext_id=jk_id, + ext_slug=jk_slug, + source_url=source_url, + name=name, + address=address, + lat=lat, + lon=lon, + house_class=house_class, + commission_year=commission_year, + commission_month=commission_month, + total_floors=total_floors, + corpus_count=corpus_count, + total_area_ha=total_area_ha, + developer_name=developer_name, + developer_url=developer_url, + developer_other_jk=developer_other_jk[:10], + rating=rating, + ratings_count=ratings_count, + text_reviews_count=text_reviews_count, + description=description_section or None, + metro_stations=metro_stations, + house_type=house_type, + raw_payload={ + "description_len": len(description_section or ""), + "body_len": len(body_text), + }, + ) + + +# ── helpers ─────────────────────────────────────────────────────────────────── + + +def _extract_coords(html: str) -> tuple[float | None, float | None]: + """Scan HTML for any `.\\d+` and `.\\d+` matching plausible city range.""" + lats = [ + float(m) + for m in re.findall(r"\b(\d{2}\.\d{4,8})\b", html) + if LAT_RANGE[0] <= float(m) <= LAT_RANGE[1] + ] + lons = [ + float(m) + for m in re.findall(r"\b(\d{2}\.\d{4,8})\b", html) + if LON_RANGE[0] <= float(m) <= LON_RANGE[1] + ] + lat = lats[0] if lats else None + lon = lons[0] if lons else None + return lat, lon + + +def _find_section_text(tree: HTMLParser, heading: str) -> str | None: + for h in tree.css("h2, h3"): + if heading.lower() in (h.text(strip=True) or "").lower(): + parts: list[str] = [] + node: Node | None = h.next + while node is not None: + tag = (node.tag or "").lower() + if tag in {"h2", "h3"}: + break + txt = node.text(strip=True) if hasattr(node, "text") else "" + if txt: + parts.append(txt) + node = node.next + return " ".join(parts).strip() or None + return None + + +def _parse_corpus_count(text: str) -> int | None: + m = RE_CORPUS_COUNT_WORD.search(text) + if not m: + return None + token = m.group(1).lower() + if token.isdigit(): + return int(token) + return _WORD_NUM.get(token) + + +def _parse_metro(text: str) -> list[JKMetroStation]: + stations: list[JKMetroStation] = [] + for m in RE_METRO_INLINE.finditer(text): + name = m.group(1).strip() + # filter common noise + if not 2 <= len(name) <= 40 or "ходьб" in name.lower() or "минут" in name.lower(): + continue + try: + stations.append(JKMetroStation(name=name, walk_min=int(m.group(2)))) + except ValueError: + continue + if len(stations) >= 5: + break + return stations + + +def _extract_jk_address(body_text: str, name: str | None) -> str | None: + """Find address line — typically right after JK name, before metro entries.""" + if name and name in body_text: + after_name = body_text.split(name, 1)[1] + # Match 'Город, ул. X / ул. Y' until first metro mention + m = re.match( + r"\s*([А-ЯЁ][^•]{5,150}?)(?:\s+[А-ЯЁ][а-яё]+\s+\d+\s*мин|\d+\s+оценок|$)", + after_name, + ) + if m: + addr = m.group(1).strip().rstrip(",").strip() + if 10 < len(addr) < 200: + return addr + return None + + +# Expose RE_JK_ID at module level for external use (e.g. SERP link → newbuilding detail) +__all__ = [ + "RE_JK_ID", + "JKMetroStation", + "YandexNewbuildingInfo", + "YandexNewbuildingScraper", +] diff --git a/tradein-mvp/backend/tests/test_yandex_newbuilding.py b/tradein-mvp/backend/tests/test_yandex_newbuilding.py new file mode 100644 index 00000000..fa17eed0 --- /dev/null +++ b/tradein-mvp/backend/tests/test_yandex_newbuilding.py @@ -0,0 +1,422 @@ +"""Unit tests for yandex_newbuilding.py — ЖК landing parser (Worker B, Wave 4). + +Reference target: ЖК Татлин (slug=tatlin, id=1592987) — comfort+, June 2023, +PRINZIP developer, rating 4.3, 1505 ratings, 353 text reviews, +coords (56.855312, 60.576668). + +All tests are offline — no network calls. Parser receives hand-crafted fixture HTML. +""" + +from __future__ import annotations + +import pytest + +from app.services.scrapers.yandex_newbuilding import ( + JKMetroStation, + YandexNewbuildingInfo, + YandexNewbuildingScraper, + _extract_coords, + _find_section_text, + _parse_corpus_count, + _parse_metro, +) + +# ── Fixture HTML ────────────────────────────────────────────────────────────── + +TATLIN_FIXTURE_HTML = """ + +ЖК «Татлин» + +

ЖК «Татлин»

+
Екатеринбург, ул. Черепанова / ул. Готвальда
+PRINZIP +
+ ЖК Кислород + ЖК Нова +
+
56.855312, 60.576668
+
Уральская 11 мин. Динамо 16 мин.
+
4.3 из 5 1505 оценок Смотреть все 353 отзыва
+

О комплексе

+

ЖК «Татлин» — три 35-этажные башни в Заречном микрорайоне Екатеринбурга. + Комплекс класса комфорт+. Введён в эксплуатацию в июне 2023 года. + Расположен на участке 1,5 га. Монолитный тип дома.

+

Расположение

+

В 5 минутах от центра

+ +""" + +# Minimal fixture — missing optional sections +MINIMAL_FIXTURE_HTML = """ + +

ЖК Минимальный

+""" + + +# ── Full happy-path test ────────────────────────────────────────────────────── + + +def test_parse_jk_full_fixture(): + """Happy path: all major fields extracted from Татлин-like HTML.""" + scraper = YandexNewbuildingScraper() + info = scraper.parse( + TATLIN_FIXTURE_HTML, + jk_slug="tatlin", + jk_id="1592987", + source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/", + ) + + assert isinstance(info, YandexNewbuildingInfo) + assert info.ext_id == "1592987" + assert info.ext_slug == "tatlin" + assert "Татлин" in (info.name or "") + + # coords + assert info.lat == pytest.approx(56.855312, abs=1e-5) + assert info.lon == pytest.approx(60.576668, abs=1e-5) + + # class + commission + assert info.house_class == "comfort_plus" + assert info.commission_year == 2023 + assert info.commission_month == "июне" + + # footprint + assert info.total_floors == 35 + assert info.corpus_count == 3 + assert info.total_area_ha == pytest.approx(1.5, abs=0.01) + + # developer + assert info.developer_name == "PRINZIP" + assert info.developer_url is not None and "prinzip" in info.developer_url + assert "ЖК Кислород" in info.developer_other_jk + assert "ЖК Нова" in info.developer_other_jk + + # reviews + assert info.rating == pytest.approx(4.3, abs=0.01) + assert info.ratings_count == 1505 + assert info.text_reviews_count == 353 + + # description present + assert info.description is not None + assert len(info.description) > 10 + + # metro + assert len(info.metro_stations) >= 1 + station_names = [s.name for s in info.metro_stations] + assert any("Уральская" in n for n in station_names) + + # house type from "монолитный" + assert info.house_type == "monolith" + + # raw_payload + assert info.raw_payload is not None + assert "body_len" in info.raw_payload + + +# ── Coord extraction tests ──────────────────────────────────────────────────── + + +def test_extract_coords_within_ekb_range(): + html = "
56.855312, 60.576668
" + lat, lon = _extract_coords(html) + assert lat == pytest.approx(56.855312, abs=1e-5) + assert lon == pytest.approx(60.576668, abs=1e-5) + + +def test_extract_coords_outside_range_returns_none(): + # Moscow coords — outside EKB range + html = "
55.7558, 37.6176
" + _lat, lon = _extract_coords(html) + # 55.75 is in LAT_RANGE but 37.61 is NOT in LON_RANGE + assert lon is None + + +def test_extract_coords_no_coords_returns_none(): + html = "
Текст без координат
" + lat, lon = _extract_coords(html) + assert lat is None + assert lon is None + + +def test_extract_coords_multiple_takes_first(): + """When multiple valid coords present, first is returned.""" + html = "
56.855312, 60.576668
56.900000, 60.600000
" + lat, lon = _extract_coords(html) + assert lat == pytest.approx(56.855312, abs=1e-5) + assert lon == pytest.approx(60.576668, abs=1e-5) + + +# ── Corpus count tests ──────────────────────────────────────────────────────── + + +def test_parse_corpus_count_word_three(): + text = "три 35-этажные башни в центре" + result = _parse_corpus_count(text) + assert result == 3 + + +def test_parse_corpus_count_digit(): + text = "5 35-этажных корпусов" + result = _parse_corpus_count(text) + assert result == 5 + + +def test_parse_corpus_count_two(): + text = "две 20-этажные башни" + result = _parse_corpus_count(text) + assert result == 2 + + +def test_parse_corpus_count_none_when_no_match(): + text = "Обычный текст без этажей" + result = _parse_corpus_count(text) + assert result is None + + +# ── House class test ────────────────────────────────────────────────────────── + + +def test_parse_house_class_comfort_plus_from_description(): + from app.services.scrapers.yandex_helpers import parse_house_class + + text = "Жилой комплекс класса комфорт+ в центре города" + result = parse_house_class(text) + assert result == "comfort_plus" + + +def test_parse_house_class_business(): + from app.services.scrapers.yandex_helpers import parse_house_class + + text = "ЖК класса бизнес" + result = parse_house_class(text) + assert result == "business" + + +# ── Commission year/month test ──────────────────────────────────────────────── + + +def test_parse_commission_year_month(): + """Full pipeline: parse() correctly extracts commission_year and commission_month.""" + scraper = YandexNewbuildingScraper() + info = scraper.parse( + TATLIN_FIXTURE_HTML, + jk_slug="tatlin", + jk_id="1592987", + source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/", + ) + assert info.commission_year == 2023 + assert info.commission_month == "июне" + + +def test_parse_commission_sdan_variant(): + """Regex supports 'сдан в эксплуатацию' phrasing as well.""" + html = """ +

ЖК Тест

+

О комплексе

+

Сдан в эксплуатацию в марте 2025 года.

+ """ + scraper = YandexNewbuildingScraper() + info = scraper.parse(html, jk_slug="test", jk_id="999", source_url="http://x") + assert info.commission_year == 2025 + assert info.commission_month == "марте" + + +# ── Rating + counts test ────────────────────────────────────────────────────── + + +def test_parse_rating_and_counts(): + scraper = YandexNewbuildingScraper() + info = scraper.parse( + TATLIN_FIXTURE_HTML, + jk_slug="tatlin", + jk_id="1592987", + source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/", + ) + assert info.rating == pytest.approx(4.3, abs=0.01) + assert info.ratings_count == 1505 + assert info.text_reviews_count == 353 + + +def test_parse_rating_comma_separator(): + """Rating '4,3 из 5' with comma handled correctly.""" + html = "
4,3 из 5 200 оценок
" + scraper = YandexNewbuildingScraper() + info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x") + assert info.rating == pytest.approx(4.3, abs=0.01) + assert info.ratings_count == 200 + + +# ── Developer link test ─────────────────────────────────────────────────────── + + +def test_developer_name_extracted_from_data_test_link(): + scraper = YandexNewbuildingScraper() + info = scraper.parse( + TATLIN_FIXTURE_HTML, + jk_slug="tatlin", + jk_id="1592987", + source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/", + ) + assert info.developer_name == "PRINZIP" + assert info.developer_url == "https://realty.yandex.ru/developer/prinzip" + + +def test_developer_absolute_url_kept_as_is(): + html = """ + АБС Групп + """ + scraper = YandexNewbuildingScraper() + info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x") + assert info.developer_name == "АБС Групп" + assert info.developer_url == "https://realty.yandex.ru/developer/abc" + + +# ── Metro stations test ─────────────────────────────────────────────────────── + + +def test_metro_stations_parsed(): + scraper = YandexNewbuildingScraper() + info = scraper.parse( + TATLIN_FIXTURE_HTML, + jk_slug="tatlin", + jk_id="1592987", + source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/", + ) + assert len(info.metro_stations) >= 1 + uralskaya = next( + (s for s in info.metro_stations if "Уральская" in s.name), None + ) + assert uralskaya is not None + assert uralskaya.walk_min == 11 + + +def test_parse_metro_direct(): + text = "Уральская 11 мин. Динамо 16 мин." + stations = _parse_metro(text) + assert len(stations) == 2 + assert stations[0].name == "Уральская" + assert stations[0].walk_min == 11 + assert stations[1].name == "Динамо" + assert stations[1].walk_min == 16 + + +def test_parse_metro_caps_at_five(): + text = ( + "Альфа 5 мин. Бета 6 мин. Гамма 7 мин. " + "Дельта 8 мин. Эпсилон 9 мин. Зета 10 мин." + ) + stations = _parse_metro(text) + assert len(stations) == 5 + + +# ── No description section test ─────────────────────────────────────────────── + + +def test_no_description_section_returns_none(): + """When no О комплексе / Расположение section, description is None.""" + scraper = YandexNewbuildingScraper() + info = scraper.parse( + MINIMAL_FIXTURE_HTML, + jk_slug="minimal", + jk_id="42", + source_url="http://example.com/", + ) + assert info.description is None + assert info.house_class is None + assert info.commission_year is None + assert info.total_floors is None + + +# ── fetch_around raises NotImplementedError ─────────────────────────────────── + + +@pytest.mark.asyncio +async def test_fetch_around_raises_not_implemented(): + scraper = YandexNewbuildingScraper() + with pytest.raises(NotImplementedError, match="JK-slug-based"): + await scraper.fetch_around(56.855, 60.576) + + +# ── _find_section_text helper ───────────────────────────────────────────────── + + +def test_find_section_text_returns_content_after_heading(): + from selectolax.parser import HTMLParser + + html = """ +

О комплексе

+

Описание комплекса здесь.

+

Ещё абзац.

+

Другой раздел

+

Не должен попасть.

+ """ + tree = HTMLParser(html) + result = _find_section_text(tree, "О комплексе") + assert result is not None + assert "Описание комплекса" in result + assert "Не должен" not in result + + +def test_find_section_text_returns_none_when_missing(): + from selectolax.parser import HTMLParser + + html = "

Текст

" + tree = HTMLParser(html) + result = _find_section_text(tree, "О комплексе") + assert result is None + + +# ── Developer other JK dedup ────────────────────────────────────────────────── + + +def test_developer_other_jk_dedup(): + """Duplicate JK names are not added twice.""" + html = """ +
+ ЖК Один + ЖК Один + ЖК Два +
+ """ + scraper = YandexNewbuildingScraper() + info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x") + assert info.developer_other_jk.count("ЖК Один") == 1 + assert "ЖК Два" in info.developer_other_jk + + +def test_developer_other_jk_capped_at_10(): + """Only first 10 other JK entries are kept.""" + links = "\n".join( + f'ЖК {i}' for i in range(15) + ) + html = f""" +
{links}
+ """ + scraper = YandexNewbuildingScraper() + info = scraper.parse(html, jk_slug="x", jk_id="1", source_url="http://x") + assert len(info.developer_other_jk) == 10 + + +# ── model defaults ──────────────────────────────────────────────────────────── + + +def test_yandex_newbuilding_info_defaults(): + """Model initialises with safe defaults for all optional fields.""" + info = YandexNewbuildingInfo(ext_id="1", ext_slug="x", source_url="http://x") + assert info.name is None + assert info.lat is None + assert info.lon is None + assert info.house_class is None + assert info.developer_other_jk == [] + assert info.metro_stations == [] + assert info.raw_payload is None + + +def test_jk_metro_station_model(): + station = JKMetroStation(name="Уральская", walk_min=11) + assert station.name == "Уральская" + assert station.walk_min == 11 + + station_no_time = JKMetroStation(name="Геологическая") + assert station_no_time.walk_min is None