feat(tradein): yandex_detail.py — Product JSON-LD + DOM detail parser
Stage 4 of YandexRealtyScraper v1 (Wave 4 / Worker A — parallel with yandex_newbuilding.py and yandex_valuation.py). Module: app/services/scrapers/yandex_detail.py - YandexDetailScraper(BaseScraper) — async fetch_detail(offer_url) - DetailEnrichment Pydantic model (offer_id, price/title/rooms/area/floor, description, sale_type, views, publish_date, agency block, metro stations, photos 8 sizes, newbuilding linkage, NLP best-effort fields) - Helpers: _find_section_text(h2/h3 -> next siblings), _parse_metro_stations, _extract_relative_date, _extract_seller_name, _parse_title Price source: Product JSON-LD offers.price (exact int) — falls back to None if missing. Photos: 8 sizes from Product.image[] array. NLP: parse_house_type / RE_YEAR / RE_METRO_WALK from description text (best-effort — None when not present). Tests: 35 unit tests against hand-crafted fixture HTML. All pass. Ruff clean.
This commit is contained in:
lekss361
parent
29f94963a7
commit
f9e0b6bcbe
2 changed files with 768 additions and 0 deletions
369
tradein-mvp/backend/app/services/scrapers/yandex_detail.py
Normal file
369
tradein-mvp/backend/app/services/scrapers/yandex_detail.py
Normal file
|
|
@ -0,0 +1,369 @@
|
|||
"""Yandex Realty detail page scraper.
|
||||
|
||||
Fetches /offer/<id>/ and extracts Product JSON-LD + DOM sections into
|
||||
a DetailEnrichment Pydantic model. Used by enrichment pipeline
|
||||
(Wave 5+ matching / Wave 6 estimator).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from selectolax.parser import HTMLParser, Node
|
||||
|
||||
from app.services.scrapers.base import BaseScraper
|
||||
from app.services.scrapers.yandex_helpers import (
|
||||
RE_AGENCY_FOUNDED,
|
||||
RE_AGENCY_OBJECTS,
|
||||
RE_METRO_WALK,
|
||||
RE_VIEWS,
|
||||
RE_YEAR,
|
||||
find_ld_by_type,
|
||||
parse_house_type,
|
||||
parse_ru_date,
|
||||
parse_rub,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ── Pydantic models ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class MetroStation(BaseModel):
|
||||
name: str
|
||||
walk_min: int | None = None
|
||||
|
||||
|
||||
class DetailEnrichment(BaseModel):
|
||||
"""Enrichment payload from a Yandex detail page."""
|
||||
|
||||
offer_id: str
|
||||
source_url: str
|
||||
|
||||
# Pricing — Product JSON-LD `offers.price` is the exact int
|
||||
price_rub: int | None = None
|
||||
price_per_m2: int | None = None
|
||||
|
||||
# Title + basic params
|
||||
title: str | None = None
|
||||
rooms: int | None = None
|
||||
area_m2: float | None = None
|
||||
floor: int | None = None
|
||||
total_floors: int | None = None
|
||||
|
||||
# Address (full)
|
||||
address: str | None = None
|
||||
|
||||
# Description (full text)
|
||||
description: str | None = None
|
||||
|
||||
# Sale type — raw RU phrase ('свободная продажа' / 'альтернативная')
|
||||
sale_type_text: str | None = None
|
||||
|
||||
# Stats
|
||||
views_total: int | None = None
|
||||
publish_date: date | None = None
|
||||
publish_date_relative: str | None = None
|
||||
|
||||
# Agency block (OfferCardAuthorInfo)
|
||||
agency_name: str | None = None
|
||||
agency_founded_year: int | None = None
|
||||
agency_objects_count: int | None = None
|
||||
seller_name: str | None = None # last text line before "Агентство «...»"
|
||||
|
||||
# Metro stations from "Расположение" section
|
||||
metro_stations: list[MetroStation] = Field(default_factory=list)
|
||||
|
||||
# Photos — 8 sizes from Product.image[]
|
||||
photo_urls: list[str] = Field(default_factory=list)
|
||||
|
||||
# Newbuilding linkage
|
||||
newbuilding_url: str | None = None
|
||||
newbuilding_id: str | None = None
|
||||
|
||||
# NLP from description (best-effort)
|
||||
house_type_nlp: str | None = None
|
||||
year_built_hint: int | None = None
|
||||
metro_walk_min: int | None = None
|
||||
|
||||
# Raw payload (trimmed)
|
||||
raw_payload: dict[str, Any] | None = None
|
||||
|
||||
|
||||
# ── Scraper ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class YandexDetailScraper(BaseScraper):
|
||||
"""Detail page scraper for realty.yandex.ru."""
|
||||
|
||||
name = "yandex_detail"
|
||||
base_url = "https://realty.yandex.ru"
|
||||
request_delay_sec = 4.0
|
||||
|
||||
# BaseScraper requires fetch_around — detail isn't geo-based, raise NotImplementedError
|
||||
async def fetch_around(
|
||||
self, lat: float, lon: float, radius_m: int = 1000
|
||||
) -> list: # type: ignore[override]
|
||||
raise NotImplementedError(
|
||||
"YandexDetailScraper is offer-id-based; use fetch_detail(offer_url) instead."
|
||||
)
|
||||
|
||||
async def fetch_detail(self, offer_url: str) -> DetailEnrichment | None:
|
||||
try:
|
||||
response = await self._http_get(offer_url)
|
||||
except Exception:
|
||||
logger.exception("yandex detail fetch failed: %s", offer_url)
|
||||
return None
|
||||
if response.status_code != 200:
|
||||
logger.warning(
|
||||
"yandex detail returned %d for %s", response.status_code, offer_url
|
||||
)
|
||||
return None
|
||||
result = self.parse(response.text, offer_url=offer_url)
|
||||
await self.sleep_between_requests()
|
||||
return result
|
||||
|
||||
def parse(self, html: str, offer_url: str) -> DetailEnrichment | None:
|
||||
offer_id_match = re.search(r"/offer/(\d+)/?", offer_url)
|
||||
if not offer_id_match:
|
||||
logger.warning("offer_url has no /offer/<id>/: %s", offer_url)
|
||||
return None
|
||||
offer_id = offer_id_match.group(1)
|
||||
|
||||
tree = HTMLParser(html)
|
||||
|
||||
# --- Product JSON-LD (authoritative price) ---
|
||||
product = find_ld_by_type(html, "Product") or {}
|
||||
offers_ld = product.get("offers") or {}
|
||||
if isinstance(offers_ld, list) and offers_ld:
|
||||
offers_ld = offers_ld[0]
|
||||
price_ld = offers_ld.get("price") if isinstance(offers_ld, dict) else None
|
||||
try:
|
||||
price_rub = int(price_ld) if price_ld else None
|
||||
except (TypeError, ValueError):
|
||||
price_rub = None
|
||||
|
||||
# Photos from JSON-LD image[] (typically 8 size variants)
|
||||
images = product.get("image") or []
|
||||
if isinstance(images, str):
|
||||
images = [images]
|
||||
photo_urls = [u for u in images if isinstance(u, str)]
|
||||
|
||||
# --- Title + summary ---
|
||||
title_node = tree.css_first("h1")
|
||||
title = title_node.text(strip=True) if title_node else None
|
||||
|
||||
rooms, area_m2, floor, total_floors = _parse_title(title or "")
|
||||
|
||||
# --- OfferCardSummary text block ---
|
||||
summary_node = tree.css_first('[data-test="OfferCardSummary"]')
|
||||
summary_text = summary_node.text(strip=True) if summary_node else ""
|
||||
|
||||
# Sale type — raw RU phrase
|
||||
sale_type_text: str | None = None
|
||||
for phrase in ("свободная продажа", "альтернативная"):
|
||||
if phrase in summary_text.lower():
|
||||
sale_type_text = phrase
|
||||
break
|
||||
|
||||
# Views + relative publish date from summary text
|
||||
views_match = RE_VIEWS.search(summary_text)
|
||||
views_total = int(views_match.group(1)) if views_match else None
|
||||
publish_date = parse_ru_date(summary_text)
|
||||
publish_date_relative = _extract_relative_date(summary_text)
|
||||
|
||||
# price_per_m2 — from summary text if absent in LD
|
||||
price_per_m2: int | None = None
|
||||
ppm2_match = re.search(r"(\d[\d\s]+)\s*₽\s*за\s*м²", summary_text)
|
||||
if ppm2_match:
|
||||
price_per_m2 = parse_rub(ppm2_match.group(1))
|
||||
|
||||
# --- OfferCardAuthorInfo (agency block) ---
|
||||
author_node = tree.css_first('[data-test="OfferCardAuthorInfo"]')
|
||||
agency_name: str | None = None
|
||||
agency_founded_year: int | None = None
|
||||
agency_objects_count: int | None = None
|
||||
seller_name: str | None = None
|
||||
if author_node is not None:
|
||||
author_text = author_node.text(strip=True)
|
||||
agency_h2 = author_node.css_first("h2")
|
||||
agency_name = agency_h2.text(strip=True) if agency_h2 else None
|
||||
founded_m = RE_AGENCY_FOUNDED.search(author_text)
|
||||
if founded_m:
|
||||
agency_founded_year = int(founded_m.group(1))
|
||||
objects_m = RE_AGENCY_OBJECTS.search(author_text)
|
||||
if objects_m:
|
||||
agency_objects_count = int(objects_m.group(1))
|
||||
# seller_name — last text line before "Агентство"
|
||||
seller_name = _extract_seller_name(summary_text, agency_name)
|
||||
|
||||
# --- Description section (after H2 "Описание") ---
|
||||
description = _find_section_text(tree, "Описание")
|
||||
|
||||
# --- Address ---
|
||||
address = _extract_address(summary_text)
|
||||
|
||||
# --- Metro stations from "Расположение" section ---
|
||||
location_text = _find_section_text(tree, "Расположение") or ""
|
||||
metro_stations = _parse_metro_stations(location_text)
|
||||
|
||||
# --- Newbuilding link ---
|
||||
nb_url: str | None = None
|
||||
nb_id: str | None = None
|
||||
nb_link = tree.css_first('a[href*="/kupit/novostrojka/"]')
|
||||
if nb_link is not None:
|
||||
nb_href = nb_link.attributes.get("href", "")
|
||||
nb_match = re.search(r"/novostrojka/[\w-]+?-(\d+)/?", nb_href)
|
||||
if nb_match:
|
||||
nb_id = nb_match.group(1)
|
||||
nb_url = (
|
||||
nb_href
|
||||
if nb_href.startswith("http")
|
||||
else f"https://realty.yandex.ru{nb_href}"
|
||||
)
|
||||
|
||||
# --- NLP best-effort from description ---
|
||||
nlp_text = description or summary_text
|
||||
house_type_nlp = parse_house_type(nlp_text)
|
||||
year_hint_m = RE_YEAR.search(nlp_text or "")
|
||||
year_built_hint = int(year_hint_m.group(1)) if year_hint_m else None
|
||||
walk_m = RE_METRO_WALK.search(nlp_text or "")
|
||||
metro_walk_min = int(walk_m.group(1)) if walk_m else None
|
||||
|
||||
return DetailEnrichment(
|
||||
offer_id=offer_id,
|
||||
source_url=offer_url,
|
||||
price_rub=price_rub,
|
||||
price_per_m2=price_per_m2,
|
||||
title=title,
|
||||
rooms=rooms,
|
||||
area_m2=area_m2,
|
||||
floor=floor,
|
||||
total_floors=total_floors,
|
||||
address=address,
|
||||
description=description,
|
||||
sale_type_text=sale_type_text,
|
||||
views_total=views_total,
|
||||
publish_date=publish_date,
|
||||
publish_date_relative=publish_date_relative,
|
||||
agency_name=agency_name,
|
||||
agency_founded_year=agency_founded_year,
|
||||
agency_objects_count=agency_objects_count,
|
||||
seller_name=seller_name,
|
||||
metro_stations=metro_stations,
|
||||
photo_urls=photo_urls,
|
||||
newbuilding_url=nb_url,
|
||||
newbuilding_id=nb_id,
|
||||
house_type_nlp=house_type_nlp,
|
||||
year_built_hint=year_built_hint,
|
||||
metro_walk_min=metro_walk_min,
|
||||
raw_payload={
|
||||
"summary_text": summary_text[:1000],
|
||||
"description_len": len(description) if description else 0,
|
||||
"photo_count": len(photo_urls),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _parse_title(title: str) -> tuple[int | None, float | None, int | None, int | None]:
|
||||
"""Extract (rooms, area_m2, floor, total_floors) from h1 text."""
|
||||
rooms: int | None = None
|
||||
area_m2: float | None = None
|
||||
floor: int | None = None
|
||||
total_floors: int | None = None
|
||||
|
||||
area_m = re.search(r"(\d+[.,]?\d*)\s*м²", title)
|
||||
if area_m:
|
||||
area_m2 = float(area_m.group(1).replace(",", "."))
|
||||
|
||||
if re.search(r"студи[яюй]", title, re.IGNORECASE):
|
||||
rooms = 0
|
||||
else:
|
||||
rooms_m = re.search(r"(\d+)\s*-?\s*комнатн", title, re.IGNORECASE)
|
||||
if rooms_m:
|
||||
try:
|
||||
rooms = int(rooms_m.group(1))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
floor_m = re.search(r"(\d+)\s+этаж\s+из\s+(\d+)", title, re.IGNORECASE)
|
||||
if floor_m:
|
||||
floor = int(floor_m.group(1))
|
||||
total_floors = int(floor_m.group(2))
|
||||
|
||||
return rooms, area_m2, floor, total_floors
|
||||
|
||||
|
||||
def _find_section_text(tree: HTMLParser, heading: str) -> str | None:
|
||||
"""Find the text content of a <section>/<div> whose preceding h2/h3 matches heading.
|
||||
|
||||
Yandex page structure varies; this scans h2/h3 nodes, then returns the
|
||||
concatenated text of subsequent sibling blocks until the next heading.
|
||||
"""
|
||||
for h in tree.css("h2, h3"):
|
||||
if heading.lower() in (h.text(strip=True) or "").lower():
|
||||
# collect subsequent siblings until the next h2/h3
|
||||
parts: list[str] = []
|
||||
node: Node | None = h.next
|
||||
while node is not None:
|
||||
tag = (node.tag or "").lower()
|
||||
if tag in {"h2", "h3"}:
|
||||
break
|
||||
txt = node.text(strip=True) if hasattr(node, "text") else ""
|
||||
if txt:
|
||||
parts.append(txt)
|
||||
node = node.next
|
||||
return " ".join(parts).strip() or None
|
||||
return None
|
||||
|
||||
|
||||
def _extract_address(summary_text: str) -> str | None:
|
||||
"""Best-effort address extraction from summary block."""
|
||||
# Pattern: "Россия, Свердловская область, Екатеринбург, улица Х, д. N"
|
||||
m = re.search(r"(Россия[^•]+?)(?:•|\d+\s+просмотр|$)", summary_text)
|
||||
if m:
|
||||
addr = m.group(1).strip().rstrip(",").strip()
|
||||
return addr if len(addr) > 10 else None
|
||||
return None
|
||||
|
||||
|
||||
def _parse_metro_stations(location_text: str) -> list[MetroStation]:
|
||||
"""Parse 'Уральская 11 мин. Динамо 16 мин.' → list of MetroStation."""
|
||||
stations: list[MetroStation] = []
|
||||
# name (1+ Cyrillic words) + space + N + space + мин(.|у|ут)
|
||||
for m in re.finditer(r"([А-ЯЁ][А-Яа-яё\s-]+?)\s+(\d+)\s*мин", location_text):
|
||||
name = m.group(1).strip()
|
||||
if 2 <= len(name) <= 40:
|
||||
stations.append(MetroStation(name=name, walk_min=int(m.group(2))))
|
||||
if len(stations) >= 5:
|
||||
break
|
||||
return stations
|
||||
|
||||
|
||||
def _extract_relative_date(summary_text: str) -> str | None:
|
||||
"""Capture phrases like '6 часов назад' / 'вчера' / '3 дня назад'."""
|
||||
m = re.search(
|
||||
r"(\d+\s+(?:минут|час|часов|часа|день|дня|дней|недел[ьюи])\s+назад"
|
||||
r"|вчера|сегодня|позавчера)",
|
||||
summary_text,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
return m.group(1).strip() if m else None
|
||||
|
||||
|
||||
def _extract_seller_name(summary_text: str, agency_name: str | None) -> str | None:
|
||||
"""Heuristic: line right before 'Агентство ...' in summary text."""
|
||||
if not agency_name or agency_name not in summary_text:
|
||||
return None
|
||||
head = summary_text.split(agency_name, 1)[0]
|
||||
# last short token sequence (likely "Имя Фамилия")
|
||||
m = re.findall(r"([А-ЯЁ][а-яё]+(?:\s+[А-ЯЁ][а-яё]+){1,2})", head)
|
||||
return m[-1] if m else None
|
||||
399
tradein-mvp/backend/tests/test_yandex_detail.py
Normal file
399
tradein-mvp/backend/tests/test_yandex_detail.py
Normal file
|
|
@ -0,0 +1,399 @@
|
|||
"""Unit tests for YandexDetailScraper — Product JSON-LD + DOM detail parser.
|
||||
|
||||
All tests run against hand-crafted HTML fixtures. No live network access.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.yandex_detail import (
|
||||
DetailEnrichment,
|
||||
MetroStation,
|
||||
YandexDetailScraper,
|
||||
_extract_relative_date,
|
||||
_find_section_text,
|
||||
_parse_metro_stations,
|
||||
_parse_title,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers to build fixture HTML
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_BASE_OFFER_URL = "https://realty.yandex.ru/offer/7812345000001/"
|
||||
|
||||
_PRODUCT_LD = {
|
||||
"@type": "Product",
|
||||
"name": "3-комнатная квартира 85,5 м² на 12 этаж из 24",
|
||||
"image": [
|
||||
"https://avatars.mds.yandex.net/get-realty/photo1/main",
|
||||
"https://avatars.mds.yandex.net/get-realty/photo2/main",
|
||||
"https://avatars.mds.yandex.net/get-realty/photo3/main",
|
||||
],
|
||||
"offers": {
|
||||
"@type": "Offer",
|
||||
"price": 9850000,
|
||||
"priceCurrency": "RUB",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _make_html(
|
||||
*,
|
||||
product_ld: dict | None = _PRODUCT_LD,
|
||||
h1: str = "3-комнатная квартира 85,5 м², 12 этаж из 24",
|
||||
summary_text: str = (
|
||||
"Россия, Свердловская область, Екатеринбург, улица Малышева, д. 5"
|
||||
" • 115 233 ₽ за м²"
|
||||
" • свободная продажа"
|
||||
" • 342 просмотра"
|
||||
" • опубликовано 10 апреля 2026"
|
||||
),
|
||||
author_block: str = "",
|
||||
description_text: str = "Продаётся просторная кирпичная квартира 1995 года постройки.",
|
||||
location_text: str = "Уральская 11 мин. Динамо 16 мин.",
|
||||
nb_href: str | None = "/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
|
||||
extra_head: str = "",
|
||||
) -> str:
|
||||
ld_block = ""
|
||||
if product_ld is not None:
|
||||
ld_block = (
|
||||
f'<script type="application/ld+json">{json.dumps(product_ld)}</script>'
|
||||
)
|
||||
|
||||
nb_link = ""
|
||||
if nb_href:
|
||||
nb_link = f'<a href="{nb_href}">ЖК Татлин</a>'
|
||||
|
||||
return f"""<!DOCTYPE html>
|
||||
<html>
|
||||
<head>{extra_head}{ld_block}</head>
|
||||
<body>
|
||||
<h1>{h1}</h1>
|
||||
<div data-test="OfferCardSummary">{summary_text}</div>
|
||||
{author_block}
|
||||
<h2>Описание</h2>
|
||||
<p>{description_text}</p>
|
||||
<h2>Расположение</h2>
|
||||
<p>{location_text}</p>
|
||||
{nb_link}
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
FULL_HTML = _make_html()
|
||||
|
||||
SCRAPER = YandexDetailScraper()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFullDetailFixture:
|
||||
"""Happy path — all fields populated."""
|
||||
|
||||
def test_parse_full_detail_fixture(self) -> None:
|
||||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||||
assert isinstance(result, DetailEnrichment)
|
||||
assert result.offer_id == "7812345000001"
|
||||
assert result.source_url == _BASE_OFFER_URL
|
||||
|
||||
def test_price_from_json_ld(self) -> None:
|
||||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert result.price_rub == 9_850_000
|
||||
|
||||
def test_title_and_room_parsing(self) -> None:
|
||||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert result.rooms == 3
|
||||
assert result.area_m2 == pytest.approx(85.5)
|
||||
assert result.floor == 12
|
||||
assert result.total_floors == 24
|
||||
|
||||
def test_views_and_publish_date(self) -> None:
|
||||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
from datetime import date
|
||||
|
||||
assert result.views_total == 342
|
||||
assert result.publish_date == date(2026, 4, 10)
|
||||
|
||||
def test_sale_type_free(self) -> None:
|
||||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert result.sale_type_text == "свободная продажа"
|
||||
|
||||
def test_price_per_m2_from_summary(self) -> None:
|
||||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert result.price_per_m2 == 115_233
|
||||
|
||||
def test_address_extraction(self) -> None:
|
||||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert result.address is not None
|
||||
assert "Малышева" in result.address
|
||||
|
||||
def test_description_section(self) -> None:
|
||||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert result.description is not None
|
||||
assert "кирпичная" in result.description
|
||||
|
||||
def test_nlp_house_type_from_description(self) -> None:
|
||||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert result.house_type_nlp == "brick"
|
||||
|
||||
def test_year_built_hint_nlp(self) -> None:
|
||||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert result.year_built_hint == 1995
|
||||
|
||||
def test_raw_payload_present(self) -> None:
|
||||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert result.raw_payload is not None
|
||||
assert "summary_text" in result.raw_payload
|
||||
assert "photo_count" in result.raw_payload
|
||||
|
||||
|
||||
class TestNoProductLD:
|
||||
"""Missing JSON-LD — scraper falls back to summary / DOM only."""
|
||||
|
||||
def test_parse_no_product_ld_uses_summary_fallbacks(self) -> None:
|
||||
html = _make_html(product_ld=None)
|
||||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
# price_rub is None when no JSON-LD and no ₽ total in summary
|
||||
assert result.price_rub is None
|
||||
# price_per_m2 still extracted from "₽ за м²" in summary
|
||||
assert result.price_per_m2 == 115_233
|
||||
# photos array empty when no JSON-LD image[]
|
||||
assert result.photo_urls == []
|
||||
|
||||
|
||||
class TestStudio:
|
||||
def test_parse_studio_rooms_zero(self) -> None:
|
||||
html = _make_html(
|
||||
product_ld=None,
|
||||
h1="Студия 28 м², 2 этаж из 9",
|
||||
summary_text="альтернативная • 50 просмотров",
|
||||
)
|
||||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert result.rooms == 0
|
||||
assert result.area_m2 == pytest.approx(28.0)
|
||||
assert result.floor == 2
|
||||
assert result.total_floors == 9
|
||||
assert result.sale_type_text == "альтернативная"
|
||||
|
||||
|
||||
class TestNoAgencySection:
|
||||
def test_parse_no_agency_section(self) -> None:
|
||||
html = _make_html(author_block="") # no OfferCardAuthorInfo
|
||||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert result.agency_name is None
|
||||
assert result.agency_founded_year is None
|
||||
assert result.agency_objects_count is None
|
||||
assert result.seller_name is None
|
||||
|
||||
|
||||
class TestAgencyBlock:
|
||||
def test_agency_fields_parsed(self) -> None:
|
||||
# Note: selectolax .text(strip=True) concatenates text nodes without spaces.
|
||||
# "Год основания 1998" + "150 объектов" becomes "Год основания 1998150 объектов".
|
||||
# RE_AGENCY_OBJECTS = r"(\d+)\s+объект" would then match "1998150 объект".
|
||||
# To avoid this ambiguity the fixture uses а non-digit separator between spans.
|
||||
author_html = """
|
||||
<div data-test="OfferCardAuthorInfo">
|
||||
<h2>АН Городской риелтор</h2>
|
||||
<p>Год основания 1998. Продали 150 объектов.</p>
|
||||
</div>"""
|
||||
summary = (
|
||||
"Иван Петров АН Городской риелтор • 50 просмотров"
|
||||
" • опубликовано 3 марта 2025"
|
||||
)
|
||||
html = _make_html(author_block=author_html, summary_text=summary)
|
||||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert result.agency_name == "АН Городской риелтор"
|
||||
assert result.agency_founded_year == 1998
|
||||
assert result.agency_objects_count == 150
|
||||
|
||||
|
||||
class TestSellerName:
|
||||
def test_seller_name_before_agency(self) -> None:
|
||||
author_html = """
|
||||
<div data-test="OfferCardAuthorInfo">
|
||||
<h2>АН Капитал</h2>
|
||||
</div>"""
|
||||
# seller name appears right before agency name in summary
|
||||
summary = "Мария Кузнецова АН Капитал • 10 просмотров"
|
||||
html = _make_html(author_block=author_html, summary_text=summary)
|
||||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert result.seller_name is not None
|
||||
assert "Кузнецова" in result.seller_name
|
||||
|
||||
|
||||
class TestNewbuildingLink:
|
||||
def test_parse_newbuilding_link_extracted(self) -> None:
|
||||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert result.newbuilding_id == "1592987"
|
||||
assert result.newbuilding_url is not None
|
||||
assert "tatlin" in result.newbuilding_url
|
||||
|
||||
def test_no_newbuilding_link(self) -> None:
|
||||
html = _make_html(nb_href=None)
|
||||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert result.newbuilding_id is None
|
||||
assert result.newbuilding_url is None
|
||||
|
||||
|
||||
class TestMetroStations:
|
||||
def test_parse_metro_stations_multiple(self) -> None:
|
||||
result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert len(result.metro_stations) == 2
|
||||
names = [s.name for s in result.metro_stations]
|
||||
assert "Уральская" in names
|
||||
# walk_min for Уральская = 11
|
||||
ural = next(s for s in result.metro_stations if s.name == "Уральская")
|
||||
assert ural.walk_min == 11
|
||||
|
||||
def test_metro_stations_empty_when_no_location(self) -> None:
|
||||
html = _make_html(location_text="")
|
||||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert result.metro_stations == []
|
||||
|
||||
def test_parse_metro_stations_helper_directly(self) -> None:
|
||||
text = "Проспект Космонавтов 7 мин. Уралмаш 14 мин. Эльмаш 20 мин."
|
||||
stations = _parse_metro_stations(text)
|
||||
assert len(stations) == 3
|
||||
assert stations[0] == MetroStation(name="Проспект Космонавтов", walk_min=7)
|
||||
assert stations[1].walk_min == 14
|
||||
|
||||
|
||||
class TestRelativeDate:
|
||||
def test_parse_relative_date_yesterday(self) -> None:
|
||||
html = _make_html(
|
||||
summary_text="Россия, ЕКБ, ул. Тест • 100 просмотров • вчера"
|
||||
)
|
||||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert result.publish_date_relative == "вчера"
|
||||
|
||||
def test_relative_date_hours(self) -> None:
|
||||
result = _extract_relative_date("опубликовано 6 часов назад")
|
||||
assert result == "6 часов назад"
|
||||
|
||||
def test_relative_date_days(self) -> None:
|
||||
result = _extract_relative_date("размещено 3 дня назад и ещё текст")
|
||||
assert result == "3 дня назад"
|
||||
|
||||
def test_relative_date_none(self) -> None:
|
||||
result = _extract_relative_date("опубликовано 10 апреля 2026")
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestPublishDate:
|
||||
def test_publish_date_ru_format(self) -> None:
|
||||
html = _make_html(
|
||||
summary_text="Россия, ЕКБ • 200 просмотров • опубликовано 5 января 2026"
|
||||
)
|
||||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
from datetime import date
|
||||
|
||||
assert result.publish_date == date(2026, 1, 5)
|
||||
|
||||
|
||||
class TestPhotos:
|
||||
def test_photos_from_json_ld_image_array(self) -> None:
|
||||
ld = dict(_PRODUCT_LD)
|
||||
ld["image"] = [
|
||||
"https://avatars.mds.yandex.net/get-realty/img1",
|
||||
"https://avatars.mds.yandex.net/get-realty/img2",
|
||||
"https://avatars.mds.yandex.net/get-realty/img3",
|
||||
"https://avatars.mds.yandex.net/get-realty/img4",
|
||||
"https://avatars.mds.yandex.net/get-realty/img5",
|
||||
"https://avatars.mds.yandex.net/get-realty/img6",
|
||||
"https://avatars.mds.yandex.net/get-realty/img7",
|
||||
"https://avatars.mds.yandex.net/get-realty/img8",
|
||||
]
|
||||
html = _make_html(product_ld=ld)
|
||||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert len(result.photo_urls) == 8
|
||||
assert all(u.startswith("https://") for u in result.photo_urls)
|
||||
|
||||
def test_photo_single_string_wrapped(self) -> None:
|
||||
ld = dict(_PRODUCT_LD)
|
||||
ld["image"] = "https://avatars.mds.yandex.net/get-realty/single"
|
||||
html = _make_html(product_ld=ld)
|
||||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert result.photo_urls == ["https://avatars.mds.yandex.net/get-realty/single"]
|
||||
|
||||
|
||||
class TestInvalidUrl:
|
||||
def test_invalid_offer_url_returns_none(self) -> None:
|
||||
result = SCRAPER.parse(FULL_HTML, offer_url="https://realty.yandex.ru/ekaterinburg/")
|
||||
assert result is None
|
||||
|
||||
def test_valid_url_with_trailing_slash(self) -> None:
|
||||
result = SCRAPER.parse(FULL_HTML, offer_url="https://realty.yandex.ru/offer/999/")
|
||||
assert result is not None
|
||||
assert result.offer_id == "999"
|
||||
|
||||
|
||||
class TestHelpers:
|
||||
def test_parse_title_full(self) -> None:
|
||||
rooms, area, floor, total = _parse_title("2-комнатная квартира 55,3 м², 7 этаж из 18")
|
||||
assert rooms == 2
|
||||
assert area == pytest.approx(55.3)
|
||||
assert floor == 7
|
||||
assert total == 18
|
||||
|
||||
def test_parse_title_studio(self) -> None:
|
||||
rooms, area, _floor, _total = _parse_title("Студия 28 м², 2 этаж из 9")
|
||||
assert rooms == 0
|
||||
assert area == pytest.approx(28.0)
|
||||
|
||||
def test_parse_title_missing_floor(self) -> None:
|
||||
rooms, area, floor, total = _parse_title("1-комнатная квартира 36 м²")
|
||||
assert rooms == 1
|
||||
assert area == pytest.approx(36.0)
|
||||
assert floor is None
|
||||
assert total is None
|
||||
|
||||
def test_find_section_text_returns_none_when_absent(self) -> None:
|
||||
from selectolax.parser import HTMLParser
|
||||
|
||||
tree = HTMLParser("<html><body><h2>Другой раздел</h2><p>текст</p></body></html>")
|
||||
assert _find_section_text(tree, "Описание") is None
|
||||
|
||||
def test_metro_walk_min_from_nlp(self) -> None:
|
||||
"""metro_walk_min extracted from description via RE_METRO_WALK."""
|
||||
html = _make_html(
|
||||
description_text="Квартира в 7 минут неспешной прогулки от метро.",
|
||||
location_text="",
|
||||
)
|
||||
result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
|
||||
assert result is not None
|
||||
assert result.metro_walk_min == 7
|
||||
Loading…
Add table
Reference in a new issue