From 3d90221fa03d34090c622934ab429a180b7498a2 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 23 May 2026 13:45:09 +0000 Subject: [PATCH] =?UTF-8?q?feat(tradein):=20yandex=5Fdetail.py=20=E2=80=94?= =?UTF-8?q?=20Product=20JSON-LD=20+=20DOM=20detail=20parser=20(#466)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 4 of YandexRealtyScraper v1. YandexDetailScraper.fetch_detail(offer_url) extracts authoritative price from Product JSON-LD (offers.price exact int) + DOM sections (description, agent, stats, metro, photos 8 sizes, newbuilding link, NLP). 35 unit tests, ruff clean. Photo URLs sanitized downstream via _safe_url CDN allowlist (avatars.mds.yandex.net). --- .../app/services/scrapers/yandex_detail.py | 369 ++++++++++++++++ .../backend/tests/test_yandex_detail.py | 399 ++++++++++++++++++ 2 files changed, 768 insertions(+) create mode 100644 tradein-mvp/backend/app/services/scrapers/yandex_detail.py create mode 100644 tradein-mvp/backend/tests/test_yandex_detail.py diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_detail.py b/tradein-mvp/backend/app/services/scrapers/yandex_detail.py new file mode 100644 index 00000000..4a4985ec --- /dev/null +++ b/tradein-mvp/backend/app/services/scrapers/yandex_detail.py @@ -0,0 +1,369 @@ +"""Yandex Realty detail page scraper. + +Fetches /offer// and extracts Product JSON-LD + DOM sections into +a DetailEnrichment Pydantic model. Used by enrichment pipeline +(Wave 5+ matching / Wave 6 estimator). +""" + +from __future__ import annotations + +import logging +import re +from datetime import date +from typing import Any + +from pydantic import BaseModel, Field +from selectolax.parser import HTMLParser, Node + +from app.services.scrapers.base import BaseScraper +from app.services.scrapers.yandex_helpers import ( + RE_AGENCY_FOUNDED, + RE_AGENCY_OBJECTS, + RE_METRO_WALK, + RE_VIEWS, + RE_YEAR, + find_ld_by_type, + parse_house_type, + parse_ru_date, + parse_rub, +) + +logger = logging.getLogger(__name__) + + +# ── Pydantic models ─────────────────────────────────────────────────────────── + + +class MetroStation(BaseModel): + name: str + walk_min: int | None = None + + +class DetailEnrichment(BaseModel): + """Enrichment payload from a Yandex detail page.""" + + offer_id: str + source_url: str + + # Pricing — Product JSON-LD `offers.price` is the exact int + price_rub: int | None = None + price_per_m2: int | None = None + + # Title + basic params + title: str | None = None + rooms: int | None = None + area_m2: float | None = None + floor: int | None = None + total_floors: int | None = None + + # Address (full) + address: str | None = None + + # Description (full text) + description: str | None = None + + # Sale type — raw RU phrase ('свободная продажа' / 'альтернативная') + sale_type_text: str | None = None + + # Stats + views_total: int | None = None + publish_date: date | None = None + publish_date_relative: str | None = None + + # Agency block (OfferCardAuthorInfo) + agency_name: str | None = None + agency_founded_year: int | None = None + agency_objects_count: int | None = None + seller_name: str | None = None # last text line before "Агентство «...»" + + # Metro stations from "Расположение" section + metro_stations: list[MetroStation] = Field(default_factory=list) + + # Photos — 8 sizes from Product.image[] + photo_urls: list[str] = Field(default_factory=list) + + # Newbuilding linkage + newbuilding_url: str | None = None + newbuilding_id: str | None = None + + # NLP from description (best-effort) + house_type_nlp: str | None = None + year_built_hint: int | None = None + metro_walk_min: int | None = None + + # Raw payload (trimmed) + raw_payload: dict[str, Any] | None = None + + +# ── Scraper ─────────────────────────────────────────────────────────────────── + + +class YandexDetailScraper(BaseScraper): + """Detail page scraper for realty.yandex.ru.""" + + name = "yandex_detail" + base_url = "https://realty.yandex.ru" + request_delay_sec = 4.0 + + # BaseScraper requires fetch_around — detail isn't geo-based, raise NotImplementedError + async def fetch_around( + self, lat: float, lon: float, radius_m: int = 1000 + ) -> list: # type: ignore[override] + raise NotImplementedError( + "YandexDetailScraper is offer-id-based; use fetch_detail(offer_url) instead." + ) + + async def fetch_detail(self, offer_url: str) -> DetailEnrichment | None: + try: + response = await self._http_get(offer_url) + except Exception: + logger.exception("yandex detail fetch failed: %s", offer_url) + return None + if response.status_code != 200: + logger.warning( + "yandex detail returned %d for %s", response.status_code, offer_url + ) + return None + result = self.parse(response.text, offer_url=offer_url) + await self.sleep_between_requests() + return result + + def parse(self, html: str, offer_url: str) -> DetailEnrichment | None: + offer_id_match = re.search(r"/offer/(\d+)/?", offer_url) + if not offer_id_match: + logger.warning("offer_url has no /offer//: %s", offer_url) + return None + offer_id = offer_id_match.group(1) + + tree = HTMLParser(html) + + # --- Product JSON-LD (authoritative price) --- + product = find_ld_by_type(html, "Product") or {} + offers_ld = product.get("offers") or {} + if isinstance(offers_ld, list) and offers_ld: + offers_ld = offers_ld[0] + price_ld = offers_ld.get("price") if isinstance(offers_ld, dict) else None + try: + price_rub = int(price_ld) if price_ld else None + except (TypeError, ValueError): + price_rub = None + + # Photos from JSON-LD image[] (typically 8 size variants) + images = product.get("image") or [] + if isinstance(images, str): + images = [images] + photo_urls = [u for u in images if isinstance(u, str)] + + # --- Title + summary --- + title_node = tree.css_first("h1") + title = title_node.text(strip=True) if title_node else None + + rooms, area_m2, floor, total_floors = _parse_title(title or "") + + # --- OfferCardSummary text block --- + summary_node = tree.css_first('[data-test="OfferCardSummary"]') + summary_text = summary_node.text(strip=True) if summary_node else "" + + # Sale type — raw RU phrase + sale_type_text: str | None = None + for phrase in ("свободная продажа", "альтернативная"): + if phrase in summary_text.lower(): + sale_type_text = phrase + break + + # Views + relative publish date from summary text + views_match = RE_VIEWS.search(summary_text) + views_total = int(views_match.group(1)) if views_match else None + publish_date = parse_ru_date(summary_text) + publish_date_relative = _extract_relative_date(summary_text) + + # price_per_m2 — from summary text if absent in LD + price_per_m2: int | None = None + ppm2_match = re.search(r"(\d[\d\s]+)\s*₽\s*за\s*м²", summary_text) + if ppm2_match: + price_per_m2 = parse_rub(ppm2_match.group(1)) + + # --- OfferCardAuthorInfo (agency block) --- + author_node = tree.css_first('[data-test="OfferCardAuthorInfo"]') + agency_name: str | None = None + agency_founded_year: int | None = None + agency_objects_count: int | None = None + seller_name: str | None = None + if author_node is not None: + author_text = author_node.text(strip=True) + agency_h2 = author_node.css_first("h2") + agency_name = agency_h2.text(strip=True) if agency_h2 else None + founded_m = RE_AGENCY_FOUNDED.search(author_text) + if founded_m: + agency_founded_year = int(founded_m.group(1)) + objects_m = RE_AGENCY_OBJECTS.search(author_text) + if objects_m: + agency_objects_count = int(objects_m.group(1)) + # seller_name — last text line before "Агентство" + seller_name = _extract_seller_name(summary_text, agency_name) + + # --- Description section (after H2 "Описание") --- + description = _find_section_text(tree, "Описание") + + # --- Address --- + address = _extract_address(summary_text) + + # --- Metro stations from "Расположение" section --- + location_text = _find_section_text(tree, "Расположение") or "" + metro_stations = _parse_metro_stations(location_text) + + # --- Newbuilding link --- + nb_url: str | None = None + nb_id: str | None = None + nb_link = tree.css_first('a[href*="/kupit/novostrojka/"]') + if nb_link is not None: + nb_href = nb_link.attributes.get("href", "") + nb_match = re.search(r"/novostrojka/[\w-]+?-(\d+)/?", nb_href) + if nb_match: + nb_id = nb_match.group(1) + nb_url = ( + nb_href + if nb_href.startswith("http") + else f"https://realty.yandex.ru{nb_href}" + ) + + # --- NLP best-effort from description --- + nlp_text = description or summary_text + house_type_nlp = parse_house_type(nlp_text) + year_hint_m = RE_YEAR.search(nlp_text or "") + year_built_hint = int(year_hint_m.group(1)) if year_hint_m else None + walk_m = RE_METRO_WALK.search(nlp_text or "") + metro_walk_min = int(walk_m.group(1)) if walk_m else None + + return DetailEnrichment( + offer_id=offer_id, + source_url=offer_url, + price_rub=price_rub, + price_per_m2=price_per_m2, + title=title, + rooms=rooms, + area_m2=area_m2, + floor=floor, + total_floors=total_floors, + address=address, + description=description, + sale_type_text=sale_type_text, + views_total=views_total, + publish_date=publish_date, + publish_date_relative=publish_date_relative, + agency_name=agency_name, + agency_founded_year=agency_founded_year, + agency_objects_count=agency_objects_count, + seller_name=seller_name, + metro_stations=metro_stations, + photo_urls=photo_urls, + newbuilding_url=nb_url, + newbuilding_id=nb_id, + house_type_nlp=house_type_nlp, + year_built_hint=year_built_hint, + metro_walk_min=metro_walk_min, + raw_payload={ + "summary_text": summary_text[:1000], + "description_len": len(description) if description else 0, + "photo_count": len(photo_urls), + }, + ) + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _parse_title(title: str) -> tuple[int | None, float | None, int | None, int | None]: + """Extract (rooms, area_m2, floor, total_floors) from h1 text.""" + rooms: int | None = None + area_m2: float | None = None + floor: int | None = None + total_floors: int | None = None + + area_m = re.search(r"(\d+[.,]?\d*)\s*м²", title) + if area_m: + area_m2 = float(area_m.group(1).replace(",", ".")) + + if re.search(r"студи[яюй]", title, re.IGNORECASE): + rooms = 0 + else: + rooms_m = re.search(r"(\d+)\s*-?\s*комнатн", title, re.IGNORECASE) + if rooms_m: + try: + rooms = int(rooms_m.group(1)) + except ValueError: + pass + + floor_m = re.search(r"(\d+)\s+этаж\s+из\s+(\d+)", title, re.IGNORECASE) + if floor_m: + floor = int(floor_m.group(1)) + total_floors = int(floor_m.group(2)) + + return rooms, area_m2, floor, total_floors + + +def _find_section_text(tree: HTMLParser, heading: str) -> str | None: + """Find the text content of a
/
whose preceding h2/h3 matches heading. + + Yandex page structure varies; this scans h2/h3 nodes, then returns the + concatenated text of subsequent sibling blocks until the next heading. + """ + for h in tree.css("h2, h3"): + if heading.lower() in (h.text(strip=True) or "").lower(): + # collect subsequent siblings until the next h2/h3 + parts: list[str] = [] + node: Node | None = h.next + while node is not None: + tag = (node.tag or "").lower() + if tag in {"h2", "h3"}: + break + txt = node.text(strip=True) if hasattr(node, "text") else "" + if txt: + parts.append(txt) + node = node.next + return " ".join(parts).strip() or None + return None + + +def _extract_address(summary_text: str) -> str | None: + """Best-effort address extraction from summary block.""" + # Pattern: "Россия, Свердловская область, Екатеринбург, улица Х, д. N" + m = re.search(r"(Россия[^•]+?)(?:•|\d+\s+просмотр|$)", summary_text) + if m: + addr = m.group(1).strip().rstrip(",").strip() + return addr if len(addr) > 10 else None + return None + + +def _parse_metro_stations(location_text: str) -> list[MetroStation]: + """Parse 'Уральская 11 мин. Динамо 16 мин.' → list of MetroStation.""" + stations: list[MetroStation] = [] + # name (1+ Cyrillic words) + space + N + space + мин(.|у|ут) + for m in re.finditer(r"([А-ЯЁ][А-Яа-яё\s-]+?)\s+(\d+)\s*мин", location_text): + name = m.group(1).strip() + if 2 <= len(name) <= 40: + stations.append(MetroStation(name=name, walk_min=int(m.group(2)))) + if len(stations) >= 5: + break + return stations + + +def _extract_relative_date(summary_text: str) -> str | None: + """Capture phrases like '6 часов назад' / 'вчера' / '3 дня назад'.""" + m = re.search( + r"(\d+\s+(?:минут|час|часов|часа|день|дня|дней|недел[ьюи])\s+назад" + r"|вчера|сегодня|позавчера)", + summary_text, + re.IGNORECASE, + ) + return m.group(1).strip() if m else None + + +def _extract_seller_name(summary_text: str, agency_name: str | None) -> str | None: + """Heuristic: line right before 'Агентство ...' in summary text.""" + if not agency_name or agency_name not in summary_text: + return None + head = summary_text.split(agency_name, 1)[0] + # last short token sequence (likely "Имя Фамилия") + m = re.findall(r"([А-ЯЁ][а-яё]+(?:\s+[А-ЯЁ][а-яё]+){1,2})", head) + return m[-1] if m else None diff --git a/tradein-mvp/backend/tests/test_yandex_detail.py b/tradein-mvp/backend/tests/test_yandex_detail.py new file mode 100644 index 00000000..f221e715 --- /dev/null +++ b/tradein-mvp/backend/tests/test_yandex_detail.py @@ -0,0 +1,399 @@ +"""Unit tests for YandexDetailScraper — Product JSON-LD + DOM detail parser. + +All tests run against hand-crafted HTML fixtures. No live network access. +""" + +from __future__ import annotations + +import json + +import pytest + +from app.services.scrapers.yandex_detail import ( + DetailEnrichment, + MetroStation, + YandexDetailScraper, + _extract_relative_date, + _find_section_text, + _parse_metro_stations, + _parse_title, +) + +# --------------------------------------------------------------------------- +# Helpers to build fixture HTML +# --------------------------------------------------------------------------- + +_BASE_OFFER_URL = "https://realty.yandex.ru/offer/7812345000001/" + +_PRODUCT_LD = { + "@type": "Product", + "name": "3-комнатная квартира 85,5 м² на 12 этаж из 24", + "image": [ + "https://avatars.mds.yandex.net/get-realty/photo1/main", + "https://avatars.mds.yandex.net/get-realty/photo2/main", + "https://avatars.mds.yandex.net/get-realty/photo3/main", + ], + "offers": { + "@type": "Offer", + "price": 9850000, + "priceCurrency": "RUB", + }, +} + + +def _make_html( + *, + product_ld: dict | None = _PRODUCT_LD, + h1: str = "3-комнатная квартира 85,5 м², 12 этаж из 24", + summary_text: str = ( + "Россия, Свердловская область, Екатеринбург, улица Малышева, д. 5" + " • 115 233 ₽ за м²" + " • свободная продажа" + " • 342 просмотра" + " • опубликовано 10 апреля 2026" + ), + author_block: str = "", + description_text: str = "Продаётся просторная кирпичная квартира 1995 года постройки.", + location_text: str = "Уральская 11 мин. Динамо 16 мин.", + nb_href: str | None = "/ekaterinburg/kupit/novostrojka/tatlin-1592987/", + extra_head: str = "", +) -> str: + ld_block = "" + if product_ld is not None: + ld_block = ( + f'' + ) + + nb_link = "" + if nb_href: + nb_link = f'ЖК Татлин' + + return f""" + +{extra_head}{ld_block} + +

{h1}

+
{summary_text}
+ {author_block} +

Описание

+

{description_text}

+

Расположение

+

{location_text}

+ {nb_link} + +""" + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +FULL_HTML = _make_html() + +SCRAPER = YandexDetailScraper() + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestFullDetailFixture: + """Happy path — all fields populated.""" + + def test_parse_full_detail_fixture(self) -> None: + result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL) + assert isinstance(result, DetailEnrichment) + assert result.offer_id == "7812345000001" + assert result.source_url == _BASE_OFFER_URL + + def test_price_from_json_ld(self) -> None: + result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL) + assert result is not None + assert result.price_rub == 9_850_000 + + def test_title_and_room_parsing(self) -> None: + result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL) + assert result is not None + assert result.rooms == 3 + assert result.area_m2 == pytest.approx(85.5) + assert result.floor == 12 + assert result.total_floors == 24 + + def test_views_and_publish_date(self) -> None: + result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL) + assert result is not None + from datetime import date + + assert result.views_total == 342 + assert result.publish_date == date(2026, 4, 10) + + def test_sale_type_free(self) -> None: + result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL) + assert result is not None + assert result.sale_type_text == "свободная продажа" + + def test_price_per_m2_from_summary(self) -> None: + result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL) + assert result is not None + assert result.price_per_m2 == 115_233 + + def test_address_extraction(self) -> None: + result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL) + assert result is not None + assert result.address is not None + assert "Малышева" in result.address + + def test_description_section(self) -> None: + result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL) + assert result is not None + assert result.description is not None + assert "кирпичная" in result.description + + def test_nlp_house_type_from_description(self) -> None: + result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL) + assert result is not None + assert result.house_type_nlp == "brick" + + def test_year_built_hint_nlp(self) -> None: + result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL) + assert result is not None + assert result.year_built_hint == 1995 + + def test_raw_payload_present(self) -> None: + result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL) + assert result is not None + assert result.raw_payload is not None + assert "summary_text" in result.raw_payload + assert "photo_count" in result.raw_payload + + +class TestNoProductLD: + """Missing JSON-LD — scraper falls back to summary / DOM only.""" + + def test_parse_no_product_ld_uses_summary_fallbacks(self) -> None: + html = _make_html(product_ld=None) + result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL) + assert result is not None + # price_rub is None when no JSON-LD and no ₽ total in summary + assert result.price_rub is None + # price_per_m2 still extracted from "₽ за м²" in summary + assert result.price_per_m2 == 115_233 + # photos array empty when no JSON-LD image[] + assert result.photo_urls == [] + + +class TestStudio: + def test_parse_studio_rooms_zero(self) -> None: + html = _make_html( + product_ld=None, + h1="Студия 28 м², 2 этаж из 9", + summary_text="альтернативная • 50 просмотров", + ) + result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL) + assert result is not None + assert result.rooms == 0 + assert result.area_m2 == pytest.approx(28.0) + assert result.floor == 2 + assert result.total_floors == 9 + assert result.sale_type_text == "альтернативная" + + +class TestNoAgencySection: + def test_parse_no_agency_section(self) -> None: + html = _make_html(author_block="") # no OfferCardAuthorInfo + result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL) + assert result is not None + assert result.agency_name is None + assert result.agency_founded_year is None + assert result.agency_objects_count is None + assert result.seller_name is None + + +class TestAgencyBlock: + def test_agency_fields_parsed(self) -> None: + # Note: selectolax .text(strip=True) concatenates text nodes without spaces. + # "Год основания 1998" + "150 объектов" becomes "Год основания 1998150 объектов". + # RE_AGENCY_OBJECTS = r"(\d+)\s+объект" would then match "1998150 объект". + # To avoid this ambiguity the fixture uses а non-digit separator between spans. + author_html = """ +
+

АН Городской риелтор

+

Год основания 1998. Продали 150 объектов.

+
""" + summary = ( + "Иван Петров АН Городской риелтор • 50 просмотров" + " • опубликовано 3 марта 2025" + ) + html = _make_html(author_block=author_html, summary_text=summary) + result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL) + assert result is not None + assert result.agency_name == "АН Городской риелтор" + assert result.agency_founded_year == 1998 + assert result.agency_objects_count == 150 + + +class TestSellerName: + def test_seller_name_before_agency(self) -> None: + author_html = """ +
+

АН Капитал

+
""" + # seller name appears right before agency name in summary + summary = "Мария Кузнецова АН Капитал • 10 просмотров" + html = _make_html(author_block=author_html, summary_text=summary) + result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL) + assert result is not None + assert result.seller_name is not None + assert "Кузнецова" in result.seller_name + + +class TestNewbuildingLink: + def test_parse_newbuilding_link_extracted(self) -> None: + result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL) + assert result is not None + assert result.newbuilding_id == "1592987" + assert result.newbuilding_url is not None + assert "tatlin" in result.newbuilding_url + + def test_no_newbuilding_link(self) -> None: + html = _make_html(nb_href=None) + result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL) + assert result is not None + assert result.newbuilding_id is None + assert result.newbuilding_url is None + + +class TestMetroStations: + def test_parse_metro_stations_multiple(self) -> None: + result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL) + assert result is not None + assert len(result.metro_stations) == 2 + names = [s.name for s in result.metro_stations] + assert "Уральская" in names + # walk_min for Уральская = 11 + ural = next(s for s in result.metro_stations if s.name == "Уральская") + assert ural.walk_min == 11 + + def test_metro_stations_empty_when_no_location(self) -> None: + html = _make_html(location_text="") + result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL) + assert result is not None + assert result.metro_stations == [] + + def test_parse_metro_stations_helper_directly(self) -> None: + text = "Проспект Космонавтов 7 мин. Уралмаш 14 мин. Эльмаш 20 мин." + stations = _parse_metro_stations(text) + assert len(stations) == 3 + assert stations[0] == MetroStation(name="Проспект Космонавтов", walk_min=7) + assert stations[1].walk_min == 14 + + +class TestRelativeDate: + def test_parse_relative_date_yesterday(self) -> None: + html = _make_html( + summary_text="Россия, ЕКБ, ул. Тест • 100 просмотров • вчера" + ) + result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL) + assert result is not None + assert result.publish_date_relative == "вчера" + + def test_relative_date_hours(self) -> None: + result = _extract_relative_date("опубликовано 6 часов назад") + assert result == "6 часов назад" + + def test_relative_date_days(self) -> None: + result = _extract_relative_date("размещено 3 дня назад и ещё текст") + assert result == "3 дня назад" + + def test_relative_date_none(self) -> None: + result = _extract_relative_date("опубликовано 10 апреля 2026") + assert result is None + + +class TestPublishDate: + def test_publish_date_ru_format(self) -> None: + html = _make_html( + summary_text="Россия, ЕКБ • 200 просмотров • опубликовано 5 января 2026" + ) + result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL) + assert result is not None + from datetime import date + + assert result.publish_date == date(2026, 1, 5) + + +class TestPhotos: + def test_photos_from_json_ld_image_array(self) -> None: + ld = dict(_PRODUCT_LD) + ld["image"] = [ + "https://avatars.mds.yandex.net/get-realty/img1", + "https://avatars.mds.yandex.net/get-realty/img2", + "https://avatars.mds.yandex.net/get-realty/img3", + "https://avatars.mds.yandex.net/get-realty/img4", + "https://avatars.mds.yandex.net/get-realty/img5", + "https://avatars.mds.yandex.net/get-realty/img6", + "https://avatars.mds.yandex.net/get-realty/img7", + "https://avatars.mds.yandex.net/get-realty/img8", + ] + html = _make_html(product_ld=ld) + result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL) + assert result is not None + assert len(result.photo_urls) == 8 + assert all(u.startswith("https://") for u in result.photo_urls) + + def test_photo_single_string_wrapped(self) -> None: + ld = dict(_PRODUCT_LD) + ld["image"] = "https://avatars.mds.yandex.net/get-realty/single" + html = _make_html(product_ld=ld) + result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL) + assert result is not None + assert result.photo_urls == ["https://avatars.mds.yandex.net/get-realty/single"] + + +class TestInvalidUrl: + def test_invalid_offer_url_returns_none(self) -> None: + result = SCRAPER.parse(FULL_HTML, offer_url="https://realty.yandex.ru/ekaterinburg/") + assert result is None + + def test_valid_url_with_trailing_slash(self) -> None: + result = SCRAPER.parse(FULL_HTML, offer_url="https://realty.yandex.ru/offer/999/") + assert result is not None + assert result.offer_id == "999" + + +class TestHelpers: + def test_parse_title_full(self) -> None: + rooms, area, floor, total = _parse_title("2-комнатная квартира 55,3 м², 7 этаж из 18") + assert rooms == 2 + assert area == pytest.approx(55.3) + assert floor == 7 + assert total == 18 + + def test_parse_title_studio(self) -> None: + rooms, area, _floor, _total = _parse_title("Студия 28 м², 2 этаж из 9") + assert rooms == 0 + assert area == pytest.approx(28.0) + + def test_parse_title_missing_floor(self) -> None: + rooms, area, floor, total = _parse_title("1-комнатная квартира 36 м²") + assert rooms == 1 + assert area == pytest.approx(36.0) + assert floor is None + assert total is None + + def test_find_section_text_returns_none_when_absent(self) -> None: + from selectolax.parser import HTMLParser + + tree = HTMLParser("

Другой раздел

текст

") + assert _find_section_text(tree, "Описание") is None + + def test_metro_walk_min_from_nlp(self) -> None: + """metro_walk_min extracted from description via RE_METRO_WALK.""" + html = _make_html( + description_text="Квартира в 7 минут неспешной прогулки от метро.", + location_text="", + ) + result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL) + assert result is not None + assert result.metro_walk_min == 7