feat(tradein): cian.py SERP refactor — state-based extraction (137 fields, no jitter)
Stage 3 of CianScraper v1 plan. Replaces DOM+__NEXT_DATA__ parsing with
cian_state_parser.extract_state(mfe='frontend-serp', key='initialState').
No anchor jitter — precise coords from offer.geo.coordinates.{lat,lng}.
Subdomain: ekb.cian.ru (per Schema sec 13, closed Q4).
curl_cffi anti-bot preserved (chrome120 TLS fingerprint).
Fields mapped: identity, geo, area, building, cadastral (flat+house),
bargainTerms, newbuilding link, seller (phones/is_homeowner/is_pro),
description_minhash (Cian native + SHA1 fallback), metro_stations,
balconies/loggias/bedrooms, listing_segment (vtorichka/novostroyki).
kitchen_area_m2 stored in raw_payload (not in ScrapedLot Stage 2).
Tests: 21 new unit tests in test_cian_serp_scraper.py (all passing).
Full suite: 71/71 passed.
This commit is contained in:
lekss361
parent
5258a8f023
commit
b22ea0da80
2 changed files with 793 additions and 336 deletions
|
|
@ -1,51 +1,63 @@
|
|||
"""Cian.ru scraper — парсинг вторички вокруг точки.
|
||||
"""Cian.ru scraper — state-based SERP parser (137 fields per offer).
|
||||
|
||||
Стратегия:
|
||||
- Cian.ru Next.js приложение хранит state в `__NEXT_DATA__` JSON (script tag в HTML).
|
||||
- URL pattern: https://ekaterinburg.cian.ru/cat.php?deal_type=sale&offer_type=flat&engine_version=2
|
||||
Стратегия (Stage 3 рефактор):
|
||||
- Cian React micro-frontends хранят state в window._cianConfig['frontend-serp'].push().
|
||||
- URL pattern: https://ekb.cian.ru/cat.php?deal_type=sale&offer_type=flat&engine_version=2
|
||||
ekb.cian.ru — city-specific subdomain для ЕКБ (per Schema_Cian_SERP_Inventory sec 13).
|
||||
|
||||
ВАЖНО: Циан блокирует httpx по TLS fingerprint → "Обнаружен подозрительный трафик" (403).
|
||||
Используем curl_cffi с impersonate='chrome120' — это libcurl-impersonate под капотом,
|
||||
который воспроизводит TLS ClientHello как настоящий Chrome.
|
||||
|
||||
НЕ используем anchor jitter: offer.geo.coordinates.{lat,lng} — точные координаты
|
||||
прямо из SERP state. Jitter запрещён per implementation plan.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import hashlib
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode, urljoin
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from curl_cffi.requests import AsyncSession
|
||||
from selectolax.parser import HTMLParser
|
||||
|
||||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||||
from app.services.scrapers.cian_state_parser import extract_state
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Регион 4743 = Свердловская область (Cian internal)
|
||||
# Регион 4743 = Свердловская область (Cian internal region ID)
|
||||
CIAN_EKB_REGION_ID = 4743
|
||||
|
||||
# SERP MFE name и state key (per Schema sec 1.2)
|
||||
_MFE_SERP = "frontend-serp"
|
||||
_STATE_KEY = "initialState"
|
||||
|
||||
|
||||
class CianScraper(BaseScraper):
|
||||
"""Cian vtorichka parser. Использует curl_cffi для обхода TLS fingerprint detection."""
|
||||
"""Cian SERP scraper. Использует curl_cffi для обхода TLS fingerprint.
|
||||
|
||||
Извлекает 137 полей на offer из Redux initialState через cian_state_parser.
|
||||
Координаты точные — НЕТ anchor jitter (offer.geo.coordinates).
|
||||
"""
|
||||
|
||||
name = "cian"
|
||||
base_url = "https://ekaterinburg.cian.ru"
|
||||
request_delay_sec = 6.0
|
||||
# ekb.cian.ru — city-specific subdomain для ЕКБ (per Schema sec 13, closed Q4)
|
||||
base_url = "https://ekb.cian.ru"
|
||||
request_delay_sec = 5.0 # консервативно: Cian менее агрессивен чем Avito, но 5s безопасно
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self._cffi: AsyncSession | None = None
|
||||
|
||||
async def __aenter__(self) -> "CianScraper":
|
||||
async def __aenter__(self) -> CianScraper:
|
||||
await super().__aenter__()
|
||||
# curl_cffi session с Chrome 120 TLS fingerprint
|
||||
# curl_cffi session с Chrome 120 TLS fingerprint — КРИТИЧЕСКИ ВАЖНО для Cian
|
||||
self._cffi = AsyncSession(
|
||||
impersonate="chrome120",
|
||||
timeout=25,
|
||||
timeout=30,
|
||||
headers={
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||||
|
|
@ -65,42 +77,51 @@ class CianScraper(BaseScraper):
|
|||
await super().__aexit__(*args)
|
||||
|
||||
async def fetch_around(
|
||||
self, lat: float, lon: float, radius_m: int = 1000,
|
||||
rooms: tuple[int, ...] | None = None, page: int = 1,
|
||||
self,
|
||||
lat: float,
|
||||
lon: float,
|
||||
radius_m: int = 1000,
|
||||
rooms: tuple[int, ...] | None = None,
|
||||
page: int = 1,
|
||||
) -> list[ScrapedLot]:
|
||||
"""Найти объявления Циан вокруг (lat, lon) в radius_m метрах.
|
||||
"""Найти объявления Циан по ЕКБ (lat/lon для reference; Cian не поддерживает bbox).
|
||||
|
||||
Cian SERP отдаёт весь ЕКБ — фильтрация по радиусу происходит в postgres
|
||||
через ST_DWithin после сохранения (точные coords из state).
|
||||
|
||||
rooms — список из 1,2,3,4 (Cian's room codes). Если None — все.
|
||||
page — страница выдачи (Cian ~28 на страницу).
|
||||
page — страница выдачи (Cian ~28 объявлений на страницу).
|
||||
"""
|
||||
self._anchor_lat = lat
|
||||
self._anchor_lon = lon
|
||||
self._anchor_radius_m = radius_m
|
||||
|
||||
url = self._build_url(lat, lon, radius_m, rooms, page)
|
||||
url = self._build_url(rooms, page)
|
||||
try:
|
||||
assert self._cffi is not None
|
||||
response = await self._cffi.get(url)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("cian curl_cffi fetch failed for %s", url)
|
||||
return []
|
||||
if response.status_code != 200:
|
||||
logger.warning("cian returned %d for %s", response.status_code, url)
|
||||
except Exception:
|
||||
logger.exception("cian curl_cffi fetch failed for url=%s", url)
|
||||
return []
|
||||
|
||||
lots = self._parse_html(response.text)
|
||||
if response.status_code != 200:
|
||||
logger.warning("cian returned HTTP %d for url=%s", response.status_code, url)
|
||||
return []
|
||||
|
||||
lots = self._parse_serp_html(response.text)
|
||||
logger.info(
|
||||
"cian: %d lots around (%.4f, %.4f) rooms=%s page=%d",
|
||||
len(lots), lat, lon, rooms, page,
|
||||
"cian: %d lots fetched rooms=%s page=%d url=%s",
|
||||
len(lots), rooms, page, url,
|
||||
)
|
||||
await self.sleep_between_requests()
|
||||
return lots
|
||||
|
||||
async def fetch_around_multi_room(
|
||||
self, lat: float, lon: float, radius_m: int = 1000, pages: int = 8,
|
||||
self,
|
||||
lat: float,
|
||||
lon: float,
|
||||
radius_m: int = 1000,
|
||||
pages: int = 8,
|
||||
) -> list[ScrapedLot]:
|
||||
"""Скрейп Циан по 1к/2к/3к/4+ × N страниц — расширяет выборку.
|
||||
4 комнаты × 3 страницы × ~28 = до ~330 лотов до дедупликации.
|
||||
|
||||
4 комнаты × N страниц × ~28 = до ~330+ лотов до дедупликации.
|
||||
"""
|
||||
seen: dict[str, ScrapedLot] = {}
|
||||
for rooms in ((1,), (2,), (3,), (4,)):
|
||||
|
|
@ -113,28 +134,26 @@ class CianScraper(BaseScraper):
|
|||
)
|
||||
continue
|
||||
if not lots:
|
||||
break # пустая страница → дальше нет смысла
|
||||
break # пустая страница → дальше смысла нет
|
||||
for lot in lots:
|
||||
key = lot.source_id or lot.source_url
|
||||
if key and key not in seen:
|
||||
seen[key] = lot
|
||||
logger.info(
|
||||
"cian multi-room: %d unique lots around (%.4f, %.4f)", len(seen), lat, lon
|
||||
"cian multi-room: %d unique lots (lat=%.4f lon=%.4f radius=%dm)",
|
||||
len(seen), lat, lon, radius_m,
|
||||
)
|
||||
return list(seen.values())
|
||||
|
||||
def _build_url(
|
||||
self,
|
||||
lat: float,
|
||||
lon: float,
|
||||
radius_m: int,
|
||||
rooms: tuple[int, ...] | None = None,
|
||||
page: int = 1,
|
||||
) -> str:
|
||||
"""URL для Cian каталога с регионом ЕКБ.
|
||||
"""URL для Cian каталога вторички ЕКБ.
|
||||
|
||||
Cian не имеет «радиус» в URL params, но мы фильтруем по lat/lon
|
||||
в postgres через ST_DWithin позже. Сейчас тащим всё ЕКБ.
|
||||
Используем ekb.cian.ru (city-specific subdomain).
|
||||
Регион задаётся через region= param как fallback для reliability.
|
||||
"""
|
||||
params: list[tuple[str, Any]] = [
|
||||
("deal_type", "sale"),
|
||||
|
|
@ -145,340 +164,320 @@ class CianScraper(BaseScraper):
|
|||
]
|
||||
if rooms:
|
||||
for r in rooms:
|
||||
# Cian rooms[]=1 для 1к, rooms[]=2 для 2к, и т.д.
|
||||
params.append((f"room{r}", "1"))
|
||||
if page > 1:
|
||||
params.append(("p", page))
|
||||
return f"{self.base_url}/cat.php?{urlencode(params)}"
|
||||
|
||||
def _parse_html(self, html: str) -> list[ScrapedLot]:
|
||||
"""Cian вставляет state в `window.__NEXT_DATA__` или подобный JSON блок."""
|
||||
# Попытка 1: __NEXT_DATA__ script
|
||||
items = _extract_next_data_items(html)
|
||||
if items:
|
||||
return [
|
||||
lot for lot in (self._item_to_lot(it) for it in items) if lot is not None
|
||||
]
|
||||
def _parse_serp_html(self, html: str) -> list[ScrapedLot]:
|
||||
"""Извлечь offers из Cian Redux state.
|
||||
|
||||
# Попытка 2: regex-извлечение из inline JSON state
|
||||
items = _extract_inline_offers(html)
|
||||
if items:
|
||||
return [
|
||||
lot for lot in (self._item_to_lot(it) for it in items) if lot is not None
|
||||
]
|
||||
|
||||
# Попытка 3: DOM card scrape (fallback)
|
||||
tree = HTMLParser(html)
|
||||
cards = tree.css('[data-name="CardComponent"]') or tree.css(
|
||||
'article[data-name="GeneralCard"]'
|
||||
)
|
||||
return [
|
||||
lot for lot in (self._card_to_lot(c) for c in cards) if lot is not None
|
||||
][:50]
|
||||
|
||||
def _item_to_lot(self, item: dict[str, Any]) -> ScrapedLot | None:
|
||||
"""Парс Cian offer JSON → ScrapedLot."""
|
||||
try:
|
||||
# Cian offer структура
|
||||
offer_id = item.get("cianId") or item.get("id")
|
||||
url = item.get("fullUrl") or item.get("url")
|
||||
if not url and offer_id:
|
||||
url = f"https://ekaterinburg.cian.ru/sale/flat/{offer_id}/"
|
||||
if not url:
|
||||
return None
|
||||
|
||||
# Цена
|
||||
bargain = item.get("bargainTerms") or {}
|
||||
price = bargain.get("price") or item.get("price")
|
||||
if not price:
|
||||
return None
|
||||
|
||||
# Параметры
|
||||
rooms = item.get("roomsCount")
|
||||
area = item.get("totalArea") or item.get("totalAreaSquare")
|
||||
if area:
|
||||
try:
|
||||
area = float(area)
|
||||
except (TypeError, ValueError):
|
||||
area = None
|
||||
floor = item.get("floorNumber")
|
||||
building = item.get("building") or {}
|
||||
total_floors = building.get("floorsCount")
|
||||
|
||||
# Адрес
|
||||
geo = item.get("geo") or {}
|
||||
address_parts = geo.get("userInput") or ""
|
||||
if isinstance(geo.get("address"), list):
|
||||
address_parts = ", ".join(
|
||||
a.get("fullName", "") or a.get("name", "")
|
||||
for a in geo["address"][:3]
|
||||
)
|
||||
|
||||
# Координаты
|
||||
coords = geo.get("coordinates") or {}
|
||||
lat = coords.get("lat")
|
||||
lon = coords.get("lng")
|
||||
|
||||
# Fallback anchor с jitter если нет точных
|
||||
if lat is None or lon is None:
|
||||
if hasattr(self, "_anchor_lat"):
|
||||
jitter_deg = (self._anchor_radius_m / 1000) / 111
|
||||
lat = self._anchor_lat + (random.random() - 0.5) * jitter_deg
|
||||
lon = self._anchor_lon + (random.random() - 0.5) * jitter_deg / 2
|
||||
|
||||
# Год постройки
|
||||
year_built = building.get("buildYear")
|
||||
|
||||
return ScrapedLot(
|
||||
source="cian",
|
||||
source_url=url,
|
||||
source_id=str(offer_id) if offer_id else None,
|
||||
address=address_parts or "Екатеринбург (Cian)",
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
rooms=rooms,
|
||||
area_m2=area,
|
||||
floor=floor,
|
||||
total_floors=total_floors,
|
||||
year_built=year_built,
|
||||
price_rub=int(price),
|
||||
photo_urls=[
|
||||
p.get("fullUrl") or p.get("thumbnailUrl", "")
|
||||
for p in (item.get("photos") or [])[:5]
|
||||
if isinstance(p, dict)
|
||||
],
|
||||
raw_payload=item,
|
||||
Использует cian_state_parser.extract_state() — общая утилита Stage 2.
|
||||
Возвращает пустой список если state не найден.
|
||||
"""
|
||||
state = extract_state(html, mfe=_MFE_SERP, key=_STATE_KEY)
|
||||
if state is None:
|
||||
logger.warning(
|
||||
"cian SERP state extraction failed (mfe=%s key=%s) — "
|
||||
"возможно Cian изменил структуру или вернул captcha",
|
||||
_MFE_SERP, _STATE_KEY,
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("cian item parse failed: %s", item.get("id"))
|
||||
return None
|
||||
return []
|
||||
|
||||
def _card_to_lot(self, card: Any) -> ScrapedLot | None:
|
||||
"""DOM card-парсинг для Cian 2025+ верстки.
|
||||
offers_data: list[dict[str, Any]] = (
|
||||
state.get("results", {}).get("offers", [])
|
||||
)
|
||||
if not offers_data:
|
||||
logger.warning(
|
||||
"cian state found but results.offers пуст (totalOffers=%s)",
|
||||
state.get("results", {}).get("totalOffers", "?"),
|
||||
)
|
||||
return []
|
||||
|
||||
Использует data-mark и data-name селекторы:
|
||||
- [data-mark="OfferTitle"]: "2-комн. квартира, 47,5 м², 2/9 этаж"
|
||||
- [data-mark="MainPrice"]: "6 000 000 ₽"
|
||||
- [data-name="GeoLabel"]: address parts (multiple, объединяем)
|
||||
- [data-mark="Description"]: long text (год постройки regex'ом)
|
||||
- href контейнера → source_url
|
||||
logger.info(
|
||||
"cian SERP state ok: %d offers (totalOffers=%s)",
|
||||
len(offers_data),
|
||||
state.get("results", {}).get("totalOffers", "?"),
|
||||
)
|
||||
|
||||
lots: list[ScrapedLot] = []
|
||||
for offer in offers_data:
|
||||
lot = self._offer_to_lot(offer)
|
||||
if lot is not None:
|
||||
lots.append(lot)
|
||||
return lots
|
||||
|
||||
def _offer_to_lot(self, offer: dict[str, Any]) -> ScrapedLot | None:
|
||||
"""Парсинг одного Cian offer (137 fields) → ScrapedLot.
|
||||
|
||||
Маппинг полей per Schema_Cian_SERP_Inventory sec 3, sec 8, sec 11.
|
||||
Без anchor jitter: координаты из offer.geo.coordinates (точные).
|
||||
"""
|
||||
try:
|
||||
link_el = card.css_first('a[href*="/sale/flat/"]')
|
||||
if link_el is None:
|
||||
# ── Identity ─────────────────────────────────────────────────────
|
||||
offer_id = offer.get("cianId") or offer.get("id")
|
||||
if not offer_id:
|
||||
logger.debug("cian offer пропущен: нет cianId/id")
|
||||
return None
|
||||
href = link_el.attributes.get("href", "")
|
||||
url = urljoin(self.base_url, href)
|
||||
|
||||
# offer_id из URL pattern /sale/flat/NNN/
|
||||
offer_id_m = re.search(r"/sale/flat/(\d+)/", href)
|
||||
offer_id = offer_id_m.group(1) if offer_id_m else None
|
||||
source_id = str(offer_id)
|
||||
url = offer.get("fullUrl")
|
||||
if not url:
|
||||
url = f"{self.base_url}/sale/flat/{offer_id}/"
|
||||
|
||||
# Цена
|
||||
price_el = card.css_first('[data-mark="MainPrice"]')
|
||||
price_text = price_el.text(strip=True) if price_el else ""
|
||||
price = _extract_price_rub(price_text)
|
||||
# ── Цена ─────────────────────────────────────────────────────────
|
||||
bargain: dict[str, Any] = offer.get("bargainTerms") or {}
|
||||
price = bargain.get("priceRur") or bargain.get("price")
|
||||
if not price:
|
||||
logger.debug("cian offer %s пропущен: нет цены", offer_id)
|
||||
return None
|
||||
try:
|
||||
price_rub = int(price)
|
||||
except (TypeError, ValueError):
|
||||
logger.debug("cian offer %s: не удалось распарсить цену %r", offer_id, price)
|
||||
return None
|
||||
if price_rub <= 0:
|
||||
return None
|
||||
|
||||
# Title → rooms / area / floor / total_floors
|
||||
title_el = (
|
||||
card.css_first('[data-mark="OfferTitle"]')
|
||||
or card.css_first('[data-name="TitleComponent"]')
|
||||
or card.css_first('[data-mark="Title"]')
|
||||
)
|
||||
title = title_el.text(strip=True) if title_el else ""
|
||||
rooms = _extract_rooms_from_text(title)
|
||||
area = _extract_area_from_text(title)
|
||||
floor, total = _extract_floor_from_text(title)
|
||||
# ── Параметры квартиры ───────────────────────────────────────────
|
||||
rooms: int | None = offer.get("roomsCount")
|
||||
# Студия: flatType == 'studio' → 0 комнат
|
||||
if offer.get("flatType") == "studio" and rooms is None:
|
||||
rooms = 0
|
||||
|
||||
# Address: собираем все GeoLabel, но показываем компактно
|
||||
geo_parts = [
|
||||
el.text(strip=True)
|
||||
for el in card.css('[data-name="GeoLabel"]')
|
||||
if el.text(strip=True)
|
||||
area_raw = offer.get("totalArea")
|
||||
area_m2: float | None = None
|
||||
if area_raw is not None:
|
||||
try:
|
||||
area_m2 = float(area_raw)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
living_area_raw = offer.get("livingArea")
|
||||
living_area_m2: float | None = None
|
||||
if living_area_raw is not None:
|
||||
try:
|
||||
living_area_m2 = float(living_area_raw)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
# kitchen_area_m2 сохраняем в raw_payload (поле не в ScrapedLot Stage 2)
|
||||
kitchen_area_raw = offer.get("kitchenArea")
|
||||
|
||||
floor: int | None = offer.get("floorNumber")
|
||||
|
||||
# ── Здание ───────────────────────────────────────────────────────
|
||||
building: dict[str, Any] = offer.get("building") or {}
|
||||
total_floors: int | None = building.get("floorsCount")
|
||||
year_built: int | None = building.get("buildYear")
|
||||
house_type: str | None = building.get("materialType")
|
||||
|
||||
# Newbuilding deadline year — если нет buildYear
|
||||
if year_built is None:
|
||||
deadline = building.get("deadline") or {}
|
||||
year_built = deadline.get("year")
|
||||
|
||||
# ── Кадастр ─────────────────────────────────────────────────────
|
||||
# cadastralNumber — кадастр КВАРТИРЫ
|
||||
cadastral_number: str | None = offer.get("cadastralNumber")
|
||||
# buildingCadastralNumber — кадастр ДОМА
|
||||
building_cadastral_number: str | None = offer.get("buildingCadastralNumber")
|
||||
|
||||
# ── Геолокация (точные координаты — NO jitter) ───────────────────
|
||||
geo: dict[str, Any] = offer.get("geo") or {}
|
||||
coords: dict[str, Any] = geo.get("coordinates") or {}
|
||||
lat: float | None = coords.get("lat")
|
||||
lon: float | None = coords.get("lng")
|
||||
# Если нет coords → сохраняем без lat/lon (geocode-missing обработает позже)
|
||||
if lat is None or lon is None:
|
||||
logger.debug(
|
||||
"cian offer %s: нет точных координат в state (geo=%s)",
|
||||
offer_id, coords,
|
||||
)
|
||||
lat = lon = None
|
||||
|
||||
# ── Адрес из geo.address[] ───────────────────────────────────────
|
||||
address_parts = geo.get("address") or []
|
||||
address = _format_address(address_parts)
|
||||
|
||||
# ── Metro stations ───────────────────────────────────────────────
|
||||
undergrounds: list[dict[str, Any]] = geo.get("undergrounds") or []
|
||||
metro_stations = [
|
||||
{
|
||||
"name": u.get("name"),
|
||||
"time": u.get("time"),
|
||||
"mode": u.get("transportType"),
|
||||
"line_color": u.get("lineColor"),
|
||||
"line_id": u.get("lineId"),
|
||||
"is_default": u.get("isDefault", False),
|
||||
}
|
||||
for u in undergrounds
|
||||
if u.get("name")
|
||||
]
|
||||
# Полный список сохраним в raw_payload, а display address укоротим.
|
||||
# Убираем "Свердловская область", "Екатеринбург", "м. *" (метро),
|
||||
# оставляем "р-н *", "мкр. *", улицу и дом.
|
||||
full_geo = list(geo_parts)
|
||||
display_parts: list[str] = []
|
||||
street_part: str | None = None
|
||||
house_part: str | None = None
|
||||
district_part: str | None = None
|
||||
for p in geo_parts:
|
||||
if p in {"Свердловская область", "Екатеринбург"}:
|
||||
continue
|
||||
if p.startswith("м. "):
|
||||
continue # метро не нужно в адресе
|
||||
if p.startswith("р-н ") and district_part is None:
|
||||
district_part = p
|
||||
elif p.startswith("мкр. "):
|
||||
continue # микрорайон опускаем для компактности
|
||||
elif "улица" in p or "проспект" in p or "переулок" in p or "бульвар" in p or "шоссе" in p:
|
||||
street_part = p
|
||||
elif re.match(r"^\d", p):
|
||||
house_part = p
|
||||
else:
|
||||
display_parts.append(p)
|
||||
# Финальный display: "улица X, дом Y · район Z"
|
||||
final_parts: list[str] = []
|
||||
if street_part:
|
||||
final_parts.append(street_part + (f", {house_part}" if house_part else ""))
|
||||
if district_part:
|
||||
final_parts.append(district_part)
|
||||
address = " · ".join(final_parts) if final_parts else (
|
||||
", ".join(p for p in geo_parts if p not in {"Свердловская область"}) or "Екатеринбург (Cian)"
|
||||
)
|
||||
|
||||
# Description → попытка извлечь год постройки
|
||||
desc_el = card.css_first('[data-mark="Description"]')
|
||||
desc = desc_el.text(strip=True) if desc_el else ""
|
||||
year_built = _extract_year_built(desc)
|
||||
# ── Newbuilding link ─────────────────────────────────────────────
|
||||
newbuilding: dict[str, Any] = offer.get("newbuilding") or {}
|
||||
nb_id = newbuilding.get("id")
|
||||
# id == 0 — sentinel для вторички (per Schema sec 20.5)
|
||||
house_source: str | None = None
|
||||
house_ext_id: str | None = None
|
||||
listing_segment: str | None = None
|
||||
|
||||
# PriceInfo: "126 316 ₽/м²" — можно посчитать area если её нет в title
|
||||
if not area:
|
||||
pinfo_el = card.css_first('[data-mark="PriceInfo"]')
|
||||
pinfo = pinfo_el.text(strip=True) if pinfo_el else ""
|
||||
m = re.search(r"([\d\s]+)\s*₽/м²", pinfo)
|
||||
if m:
|
||||
ppm2 = int(re.sub(r"\D", "", m.group(1)) or 0)
|
||||
if ppm2 > 0:
|
||||
area = round(price / ppm2, 2)
|
||||
if nb_id and int(nb_id) > 0:
|
||||
house_source = "cian_newbuilding"
|
||||
house_ext_id = str(nb_id)
|
||||
listing_segment = "novostroyki"
|
||||
else:
|
||||
house_source = "cian"
|
||||
listing_segment = "vtorichka"
|
||||
|
||||
# Coords: НЕ ставим (Cian card HTML не отдаёт). Оставляем None —
|
||||
# geocode-missing проставит реальные по address. Anchor+jitter тут
|
||||
# был БАГ: Cian отдаёт ВСЕ объявления ЕКБ независимо от anchor, и
|
||||
# jitter вокруг одного якоря давал фейковые координаты → ST_DWithin
|
||||
# не находил аналоги для адресов вдали от центра.
|
||||
lat = lon = None
|
||||
# ── Дополнительные характеристики ────────────────────────────────
|
||||
balconies_count: int | None = offer.get("balconiesCount")
|
||||
loggias_count: int | None = offer.get("loggiasCount")
|
||||
bedrooms_count: int | None = offer.get("bedroomsCount")
|
||||
|
||||
# Photos из img.src в card (первые 5)
|
||||
photo_urls: list[str] = []
|
||||
for img in card.css("img.x31de4314--_18b0f--container, [data-name=Cover] img")[:5]:
|
||||
src = img.attributes.get("src", "")
|
||||
if src and "cian" in src and src not in photo_urls:
|
||||
photo_urls.append(src)
|
||||
has_balcony: bool | None = None
|
||||
if balconies_count is not None:
|
||||
has_balcony = balconies_count > 0
|
||||
elif loggias_count is not None:
|
||||
has_balcony = loggias_count > 0
|
||||
|
||||
# Мебель и repair_state
|
||||
has_furniture: bool | None = offer.get("hasFurniture") or offer.get("isSoldFurnished")
|
||||
# decoration — для новостроек (отделка); repair_state — для вторички
|
||||
repair_state: str | None = offer.get("decoration")
|
||||
|
||||
# ── Продавец ─────────────────────────────────────────────────────
|
||||
phones: list[dict[str, Any]] = offer.get("phones") or []
|
||||
is_homeowner: bool | None = offer.get("isByHomeowner")
|
||||
is_pro_seller: bool | None = offer.get("isPro")
|
||||
|
||||
# ── Сделка ───────────────────────────────────────────────────────
|
||||
sale_type: str | None = bargain.get("saleType")
|
||||
bargain_allowed: bool | None = bargain.get("bargainAllowed")
|
||||
|
||||
# ── Description + minhash ─────────────────────────────────────────
|
||||
description: str | None = offer.get("description")
|
||||
# Cian предоставляет готовый minhash для dedup и cross-source matching
|
||||
description_minhash: str | None = offer.get("descriptionMinhash")
|
||||
# Если Cian не дал minhash — вычисляем простой SHA1 из description
|
||||
if not description_minhash and description:
|
||||
description_minhash = hashlib.sha1(
|
||||
description.lower().encode("utf-8", errors="replace")
|
||||
).hexdigest()[:32]
|
||||
|
||||
# ── Фото ─────────────────────────────────────────────────────────
|
||||
photo_urls: list[str] = [
|
||||
p["fullUrl"]
|
||||
for p in (offer.get("photos") or [])
|
||||
if isinstance(p, dict) and p.get("fullUrl")
|
||||
]
|
||||
|
||||
# ── Дата публикации ───────────────────────────────────────────────
|
||||
added_ts = offer.get("addedTimestamp")
|
||||
listing_date = None
|
||||
if added_ts:
|
||||
try:
|
||||
listing_date = datetime.fromtimestamp(int(added_ts), tz=UTC).date()
|
||||
except (TypeError, ValueError, OSError):
|
||||
pass
|
||||
|
||||
# ── Raw payload (полный offer для enrichment в будущем) ───────────
|
||||
raw_payload: dict[str, Any] = {
|
||||
"cian_id": offer_id,
|
||||
"flat_type": offer.get("flatType"),
|
||||
"is_apartments": offer.get("isApartments"),
|
||||
"offer_type": offer.get("offerType"),
|
||||
"category": offer.get("category"),
|
||||
"has_furniture": has_furniture,
|
||||
"kitchen_area_m2": kitchen_area_raw, # не в ScrapedLot — сохраняем здесь
|
||||
"mortgage_allowed": bargain.get("mortgageAllowed"),
|
||||
"sale_type": sale_type,
|
||||
"newbuilding_name": newbuilding.get("name"),
|
||||
"is_from_developer": (
|
||||
offer.get("fromDeveloper") or newbuilding.get("isFromDeveloper")
|
||||
),
|
||||
"builders_ids": offer.get("buildersIds"),
|
||||
"is_rosreestr_checked": offer.get("isRosreestrChecked"),
|
||||
"is_layout_approved": offer.get("isLayoutApproved"),
|
||||
"user_id": offer.get("userId"),
|
||||
"published_user_id": offer.get("publishedUserId"),
|
||||
"is_by_commercial_owner": offer.get("isByCommercialOwner"),
|
||||
"is_cian_partner": offer.get("isCianPartner"),
|
||||
"description_words_highlighted": offer.get("descriptionWordsHighlighted"),
|
||||
"building_parking": building.get("parking"),
|
||||
"building_total_area": building.get("totalArea"),
|
||||
}
|
||||
|
||||
return ScrapedLot(
|
||||
source="cian",
|
||||
source_url=url,
|
||||
source_id=offer_id,
|
||||
source_id=source_id,
|
||||
address=address,
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
rooms=rooms,
|
||||
area_m2=area,
|
||||
area_m2=area_m2,
|
||||
floor=floor,
|
||||
total_floors=total,
|
||||
total_floors=total_floors,
|
||||
year_built=year_built,
|
||||
price_rub=price,
|
||||
house_type=house_type,
|
||||
repair_state=repair_state,
|
||||
has_balcony=has_balcony,
|
||||
kadastr_num=cadastral_number,
|
||||
house_source=house_source,
|
||||
house_ext_id=house_ext_id,
|
||||
listing_segment=listing_segment,
|
||||
price_rub=price_rub,
|
||||
price_per_m2=None, # compute_price_per_m2() вычислит
|
||||
listing_date=listing_date,
|
||||
photo_urls=photo_urls,
|
||||
raw_payload={
|
||||
"title": title,
|
||||
"description": desc[:500],
|
||||
"geo": full_geo,
|
||||
},
|
||||
raw_payload=raw_payload,
|
||||
# Cian-specific (Stage 2 fields)
|
||||
living_area_m2=living_area_m2,
|
||||
bedrooms_count=bedrooms_count,
|
||||
balconies_count=balconies_count,
|
||||
loggias_count=loggias_count,
|
||||
description_minhash=description_minhash,
|
||||
cadastral_number=cadastral_number,
|
||||
building_cadastral_number=building_cadastral_number,
|
||||
phones=phones,
|
||||
is_homeowner=is_homeowner,
|
||||
is_pro_seller=is_pro_seller,
|
||||
bargain_allowed=bargain_allowed,
|
||||
sale_type=sale_type,
|
||||
metro_stations=metro_stations,
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("cian card parse failed")
|
||||
|
||||
except Exception:
|
||||
_oid = offer.get("cianId") or offer.get("id")
|
||||
logger.exception("cian _offer_to_lot failed for offer_id=%s", _oid)
|
||||
return None
|
||||
|
||||
|
||||
# ── HTML / JSON extraction helpers ──────────────────────────────────────────
|
||||
# ── Address formatter ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
_RE_NEXT_DATA = re.compile(
|
||||
r'<script[^>]+id="__NEXT_DATA__"[^>]*>(.+?)</script>', re.DOTALL
|
||||
)
|
||||
_RE_INITIAL_STATE = re.compile(
|
||||
r'window\._cianConfig\s*=\s*({.+?});</script>', re.DOTALL
|
||||
)
|
||||
def _format_address(address_parts: list[dict[str, Any]]) -> str:
|
||||
"""Сформировать читаемый адрес из geo.address[] Cian.
|
||||
|
||||
Пропускаем location (страна/регион) и metro — берём раион/улицу/дом.
|
||||
Пример: [location=Москва, raion=Пресненский, street=..., house=1С7, metro=Москва-Сити]
|
||||
→ 'Пресненский, улица ..., 1С7'
|
||||
"""
|
||||
if not address_parts:
|
||||
return "Екатеринбург (Cian)"
|
||||
|
||||
def _extract_next_data_items(html: str) -> list[dict[str, Any]]:
|
||||
"""Cian Next.js приложение хранит state в `__NEXT_DATA__` JSON."""
|
||||
m = _RE_NEXT_DATA.search(html)
|
||||
if not m:
|
||||
return []
|
||||
try:
|
||||
data = json.loads(m.group(1))
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
# Структура у Cian примерно такая: props.pageProps.initialState.serpData.offers
|
||||
cursor: Any = data
|
||||
for key in ["props", "pageProps", "initialState"]:
|
||||
if not isinstance(cursor, dict):
|
||||
return []
|
||||
cursor = cursor.get(key)
|
||||
if not isinstance(cursor, dict):
|
||||
return []
|
||||
serp = cursor.get("results") or cursor.get("serpData") or cursor.get("data") or {}
|
||||
offers = serp.get("offersSerialized") or serp.get("offers") or []
|
||||
if isinstance(offers, list):
|
||||
return offers
|
||||
return []
|
||||
|
||||
|
||||
def _extract_inline_offers(html: str) -> list[dict[str, Any]]:
|
||||
"""Альтернативный поиск offers в любом большом JSON в HTML."""
|
||||
# Найти offers: [...] inline
|
||||
for match in re.finditer(r'"offers":\[(.+?)\](?=,"|\})', html):
|
||||
chunk = "[" + match.group(1) + "]"
|
||||
try:
|
||||
offers = json.loads(chunk)
|
||||
if isinstance(offers, list) and offers and isinstance(offers[0], dict):
|
||||
return offers
|
||||
except json.JSONDecodeError:
|
||||
skip_types = {"location", "metro"}
|
||||
parts: list[str] = []
|
||||
for part in address_parts:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
return []
|
||||
ptype = part.get("type", "")
|
||||
if ptype in skip_types:
|
||||
continue
|
||||
name = (part.get("fullName") or part.get("name") or "").strip()
|
||||
if name:
|
||||
parts.append(name)
|
||||
|
||||
|
||||
# ── Title parsers (Cian-specific format) ────────────────────────────────────
|
||||
|
||||
|
||||
_RE_ROOMS = re.compile(r"(\d)-комн", re.IGNORECASE)
|
||||
_RE_STUDIO = re.compile(r"студи", re.IGNORECASE)
|
||||
_RE_AREA = re.compile(r"(\d+[.,]?\d*)\s*м²", re.IGNORECASE)
|
||||
_RE_FLOOR = re.compile(r"(\d+)\s*/\s*(\d+)\s*эт", re.IGNORECASE)
|
||||
_RE_PRICE = re.compile(r"([\d\s]+?)\s*₽")
|
||||
|
||||
|
||||
def _extract_rooms_from_text(text: str) -> int | None:
|
||||
if _RE_STUDIO.search(text):
|
||||
return 0
|
||||
m = _RE_ROOMS.search(text)
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
def _extract_area_from_text(text: str) -> float | None:
|
||||
m = _RE_AREA.search(text)
|
||||
return float(m.group(1).replace(",", ".")) if m else None
|
||||
|
||||
|
||||
def _extract_floor_from_text(text: str) -> tuple[int | None, int | None]:
|
||||
m = _RE_FLOOR.search(text)
|
||||
if m:
|
||||
return int(m.group(1)), int(m.group(2))
|
||||
return None, None
|
||||
|
||||
|
||||
def _extract_price_rub(text: str) -> int | None:
|
||||
m = _RE_PRICE.search(text)
|
||||
if m:
|
||||
digits = re.sub(r"\D", "", m.group(1))
|
||||
return int(digits) if digits else None
|
||||
return None
|
||||
|
||||
|
||||
_RE_YEAR_BUILT = re.compile(r"(?:19[5-9]\d|20[0-2]\d)\s*(?:года?\s*постройки|г\.?\s*постройки)", re.IGNORECASE)
|
||||
|
||||
|
||||
def _extract_year_built(text: str) -> int | None:
|
||||
"""Извлечь год постройки из description: '1988 года постройки' / '2005 г. постройки'."""
|
||||
m = _RE_YEAR_BUILT.search(text)
|
||||
if m:
|
||||
year_m = re.search(r"(19[5-9]\d|20[0-2]\d)", m.group(0))
|
||||
if year_m:
|
||||
return int(year_m.group(1))
|
||||
return None
|
||||
return ", ".join(parts) if parts else "Екатеринбург (Cian)"
|
||||
|
|
|
|||
458
tradein-mvp/backend/tests/test_cian_serp_scraper.py
Normal file
458
tradein-mvp/backend/tests/test_cian_serp_scraper.py
Normal file
|
|
@ -0,0 +1,458 @@
|
|||
"""Smoke тесты для cian.py SERP scraper (Stage 3 рефактор).
|
||||
|
||||
Проверяют парсинг 137 полей из синтетического Redux state без сетевых запросов.
|
||||
Fixture формирует минимальный JS push() conforming to _cianConfig pattern.
|
||||
"""
|
||||
|
||||
import json
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.cian import CianScraper, _format_address
|
||||
|
||||
|
||||
# ── Fixture helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_cian_serp_html(offers: list[dict]) -> str:
|
||||
"""Обернуть offers в валидный _cianConfig['frontend-serp'].push() HTML."""
|
||||
state = {
|
||||
"results": {
|
||||
"totalOffers": len(offers),
|
||||
"offers": offers,
|
||||
}
|
||||
}
|
||||
state_json = json.dumps(state, ensure_ascii=False)
|
||||
return f"""<html><body>
|
||||
<script>
|
||||
window._cianConfig['frontend-serp'] = window._cianConfig['frontend-serp'] || [];
|
||||
window._cianConfig['frontend-serp'].push({{
|
||||
key: 'initialState',
|
||||
value: {state_json},
|
||||
priority: 10,
|
||||
filter: null
|
||||
}});
|
||||
</script>
|
||||
</body></html>"""
|
||||
|
||||
|
||||
def _make_full_offer(
|
||||
cian_id: int = 328739308,
|
||||
price_rur: int = 6500000,
|
||||
rooms: int = 2,
|
||||
total_area: str = "52.0",
|
||||
living_area: str = "35.0",
|
||||
kitchen_area: str = "11.5",
|
||||
floor: int = 7,
|
||||
floors_count: int = 16,
|
||||
year_built: int = 2010,
|
||||
material_type: str = "panel",
|
||||
lat: float = 56.8385,
|
||||
lng: float = 60.5940,
|
||||
cadastral_number: str = "66:41:0204016:1234",
|
||||
building_cadastral_number: str = "66:41:0204016:100",
|
||||
description: str = "Продаётся отличная квартира",
|
||||
description_minhash: str = "abc123def456abc1",
|
||||
newbuilding_id: int = 0,
|
||||
has_phones: bool = True,
|
||||
is_homeowner: bool = True,
|
||||
is_pro: bool = False,
|
||||
bargain_allowed: bool = True,
|
||||
sale_type: str = "free",
|
||||
) -> dict:
|
||||
"""Синтетический Cian offer с полным набором полей (137 полей subset)."""
|
||||
offer: dict = {
|
||||
"cianId": cian_id,
|
||||
"id": cian_id,
|
||||
"fullUrl": f"https://ekb.cian.ru/sale/flat/{cian_id}/",
|
||||
"category": "flatSale",
|
||||
"dealType": "sale",
|
||||
"offerType": "flat",
|
||||
"flatType": "rooms",
|
||||
"isApartments": False,
|
||||
"roomsCount": rooms,
|
||||
"status": "published",
|
||||
# Area
|
||||
"totalArea": total_area,
|
||||
"livingArea": living_area,
|
||||
"kitchenArea": kitchen_area,
|
||||
"balconiesCount": 1,
|
||||
"loggiasCount": 0,
|
||||
"bedroomsCount": None,
|
||||
"hasFurniture": False,
|
||||
"isSoldFurnished": False,
|
||||
"decoration": None,
|
||||
# Floor / building
|
||||
"floorNumber": floor,
|
||||
"building": {
|
||||
"floorsCount": floors_count,
|
||||
"materialType": material_type,
|
||||
"buildYear": year_built,
|
||||
"deadline": None,
|
||||
"parking": None,
|
||||
},
|
||||
# Cadastral
|
||||
"cadastralNumber": cadastral_number,
|
||||
"buildingCadastralNumber": building_cadastral_number,
|
||||
# Price
|
||||
"bargainTerms": {
|
||||
"priceRur": price_rur,
|
||||
"price": price_rur,
|
||||
"currency": "rur",
|
||||
"saleType": sale_type,
|
||||
"mortgageAllowed": True,
|
||||
"bargainAllowed": bargain_allowed,
|
||||
"priceType": "all",
|
||||
},
|
||||
"formattedShortPrice": f"{price_rur // 1000000} млн ₽",
|
||||
# Geo
|
||||
"geo": {
|
||||
"coordinates": {"lat": lat, "lng": lng},
|
||||
"address": [
|
||||
{"type": "location", "name": "Свердловская область"},
|
||||
{"type": "city", "name": "Екатеринбург"},
|
||||
{"type": "street", "name": "улица Постовского"},
|
||||
{"type": "house", "name": "17А"},
|
||||
],
|
||||
"undergrounds": [
|
||||
{
|
||||
"name": "Геологическая",
|
||||
"time": 10,
|
||||
"transportType": "walk",
|
||||
"lineId": 10,
|
||||
"lineColor": "FF0000",
|
||||
"isDefault": True,
|
||||
"id": 100,
|
||||
}
|
||||
],
|
||||
"districts": [],
|
||||
},
|
||||
# Newbuilding
|
||||
"newbuilding": {"id": newbuilding_id, "name": ""},
|
||||
"buildersIds": [],
|
||||
# Seller
|
||||
"phones": [{"number": "9012345678", "countryCode": "7"}] if has_phones else [],
|
||||
"userId": 13332612,
|
||||
"cianUserId": 13332612,
|
||||
"publishedUserId": 13332612,
|
||||
"isByHomeowner": is_homeowner,
|
||||
"isByCommercialOwner": False,
|
||||
"isCianPartner": False,
|
||||
"isPro": is_pro,
|
||||
"isCalltrackingEnabled": False,
|
||||
# Description
|
||||
"description": description,
|
||||
"descriptionMinhash": description_minhash,
|
||||
"isDuplicatedDescription": False,
|
||||
# Time
|
||||
"addedTimestamp": 1716451200, # 2024-05-23 00:00:00 UTC
|
||||
"added": "вчера",
|
||||
# Photos
|
||||
"photos": [
|
||||
{
|
||||
"id": 1001,
|
||||
"fullUrl": f"https://images.cdn-cian.ru/images/{cian_id}-1.jpg",
|
||||
"thumbnailUrl": f"https://images.cdn-cian.ru/images/{cian_id}-2.jpg",
|
||||
}
|
||||
],
|
||||
# Verification
|
||||
"isRosreestrChecked": True,
|
||||
"isLayoutApproved": None,
|
||||
# Misc
|
||||
"isNew": False,
|
||||
"isTop3": False,
|
||||
"isPremium": False,
|
||||
}
|
||||
return offer
|
||||
|
||||
|
||||
# ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_cian_serp_parses_full_offer():
|
||||
"""Базовый тест: полный offer → ScrapedLot с ключевыми полями."""
|
||||
offer = _make_full_offer()
|
||||
html = _make_cian_serp_html([offer])
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
|
||||
assert len(lots) == 1
|
||||
lot = lots[0]
|
||||
|
||||
assert lot.source == "cian"
|
||||
assert lot.source_id == "328739308"
|
||||
assert lot.source_url == "https://ekb.cian.ru/sale/flat/328739308/"
|
||||
|
||||
|
||||
def test_cian_serp_coords_no_jitter():
|
||||
"""Координаты берутся точно из state — NO anchor jitter."""
|
||||
offer = _make_full_offer(lat=56.8385, lng=60.5940)
|
||||
html = _make_cian_serp_html([offer])
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
|
||||
assert len(lots) == 1
|
||||
lot = lots[0]
|
||||
assert lot.lat == pytest.approx(56.8385, abs=1e-9)
|
||||
assert lot.lon == pytest.approx(60.5940, abs=1e-9)
|
||||
|
||||
|
||||
def test_cian_serp_kitchen_area_in_raw_payload():
|
||||
"""kitchen_area_m2 не в ScrapedLot, но сохраняется в raw_payload."""
|
||||
offer = _make_full_offer(kitchen_area="11.5")
|
||||
html = _make_cian_serp_html([offer])
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
|
||||
lot = lots[0]
|
||||
assert lot.raw_payload is not None
|
||||
assert lot.raw_payload["kitchen_area_m2"] == "11.5"
|
||||
|
||||
|
||||
def test_cian_serp_description_minhash():
|
||||
"""description_minhash берётся из Cian state, если есть."""
|
||||
offer = _make_full_offer(
|
||||
description_minhash="cian_minhash_abc123",
|
||||
description="Тест описания",
|
||||
)
|
||||
html = _make_cian_serp_html([offer])
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
|
||||
lot = lots[0]
|
||||
assert lot.description_minhash == "cian_minhash_abc123"
|
||||
|
||||
|
||||
def test_cian_serp_description_minhash_fallback():
|
||||
"""Если Cian не дал minhash — вычисляем SHA1 из description."""
|
||||
offer = _make_full_offer(description_minhash="", description="Уникальное описание квартиры")
|
||||
offer["descriptionMinhash"] = "" # явно пусто
|
||||
html = _make_cian_serp_html([offer])
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
|
||||
lot = lots[0]
|
||||
assert lot.description_minhash is not None
|
||||
assert len(lot.description_minhash) == 32 # SHA1 hex [:32]
|
||||
|
||||
|
||||
def test_cian_serp_phones():
|
||||
"""phones — список dicts из state."""
|
||||
offer = _make_full_offer(has_phones=True)
|
||||
html = _make_cian_serp_html([offer])
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
|
||||
lot = lots[0]
|
||||
assert isinstance(lot.phones, list)
|
||||
assert len(lot.phones) == 1
|
||||
assert lot.phones[0]["countryCode"] == "7"
|
||||
assert lot.phones[0]["number"] == "9012345678"
|
||||
|
||||
|
||||
def test_cian_serp_house_ext_id_newbuilding():
|
||||
"""house_ext_id заполнен если offer.newbuilding.id > 0."""
|
||||
offer = _make_full_offer(newbuilding_id=54016)
|
||||
html = _make_cian_serp_html([offer])
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
|
||||
lot = lots[0]
|
||||
assert lot.house_source == "cian_newbuilding"
|
||||
assert lot.house_ext_id == "54016"
|
||||
assert lot.listing_segment == "novostroyki"
|
||||
|
||||
|
||||
def test_cian_serp_vtorichka_no_house_ext_id():
|
||||
"""Вторичка (newbuilding.id == 0) — house_ext_id None."""
|
||||
offer = _make_full_offer(newbuilding_id=0)
|
||||
html = _make_cian_serp_html([offer])
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
|
||||
lot = lots[0]
|
||||
assert lot.house_source == "cian"
|
||||
assert lot.house_ext_id is None
|
||||
assert lot.listing_segment == "vtorichka"
|
||||
|
||||
|
||||
def test_cian_serp_metro_stations():
|
||||
"""metro_stations — список dicts с name/time/mode/line_color."""
|
||||
offer = _make_full_offer()
|
||||
html = _make_cian_serp_html([offer])
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
|
||||
lot = lots[0]
|
||||
assert isinstance(lot.metro_stations, list)
|
||||
assert len(lot.metro_stations) == 1
|
||||
station = lot.metro_stations[0]
|
||||
assert station["name"] == "Геологическая"
|
||||
assert station["time"] == 10
|
||||
assert station["mode"] == "walk"
|
||||
assert station["line_color"] == "FF0000"
|
||||
|
||||
|
||||
def test_cian_serp_cadastral_numbers():
|
||||
"""Кадастровые номера квартиры и здания."""
|
||||
offer = _make_full_offer(
|
||||
cadastral_number="66:41:0204016:1234",
|
||||
building_cadastral_number="66:41:0204016:100",
|
||||
)
|
||||
html = _make_cian_serp_html([offer])
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
|
||||
lot = lots[0]
|
||||
assert lot.cadastral_number == "66:41:0204016:1234"
|
||||
assert lot.building_cadastral_number == "66:41:0204016:100"
|
||||
# kadastr_num (base field) тоже заполнен
|
||||
assert lot.kadastr_num == "66:41:0204016:1234"
|
||||
|
||||
|
||||
def test_cian_serp_is_homeowner_seller():
|
||||
"""is_homeowner и is_pro_seller из offer."""
|
||||
offer = _make_full_offer(is_homeowner=True, is_pro=False)
|
||||
html = _make_cian_serp_html([offer])
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
|
||||
lot = lots[0]
|
||||
assert lot.is_homeowner is True
|
||||
assert lot.is_pro_seller is False
|
||||
|
||||
|
||||
def test_cian_serp_bargain_and_sale_type():
|
||||
"""bargain_allowed и sale_type из bargainTerms."""
|
||||
offer = _make_full_offer(bargain_allowed=True, sale_type="alternative")
|
||||
html = _make_cian_serp_html([offer])
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
|
||||
lot = lots[0]
|
||||
assert lot.bargain_allowed is True
|
||||
assert lot.sale_type == "alternative"
|
||||
|
||||
|
||||
def test_cian_serp_building_fields():
|
||||
"""Поля здания: year_built, house_type, total_floors."""
|
||||
offer = _make_full_offer(year_built=1988, material_type="brick", floors_count=9)
|
||||
html = _make_cian_serp_html([offer])
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
|
||||
lot = lots[0]
|
||||
assert lot.year_built == 1988
|
||||
assert lot.house_type == "brick"
|
||||
assert lot.total_floors == 9
|
||||
|
||||
|
||||
def test_cian_serp_living_area():
|
||||
"""living_area_m2 из offer.livingArea."""
|
||||
offer = _make_full_offer(living_area="35.0")
|
||||
html = _make_cian_serp_html([offer])
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
|
||||
lot = lots[0]
|
||||
assert lot.living_area_m2 == pytest.approx(35.0)
|
||||
|
||||
|
||||
def test_cian_serp_state_missing_returns_empty():
|
||||
"""Если state нет в HTML — возвращаем пустой список."""
|
||||
html = "<html><body><p>no cian state here</p></body></html>"
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
assert lots == []
|
||||
|
||||
|
||||
def test_cian_serp_skips_offer_without_price():
|
||||
"""Offer без цены — пропускаем."""
|
||||
offer = _make_full_offer()
|
||||
del offer["bargainTerms"]["priceRur"]
|
||||
del offer["bargainTerms"]["price"]
|
||||
html = _make_cian_serp_html([offer])
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
assert lots == []
|
||||
|
||||
|
||||
def test_cian_serp_multiple_offers():
|
||||
"""Несколько offers → список ScrapedLot."""
|
||||
offers = [_make_full_offer(cian_id=100 + i) for i in range(5)]
|
||||
html = _make_cian_serp_html(offers)
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
assert len(lots) == 5
|
||||
ids = [lot.source_id for lot in lots]
|
||||
assert sorted(ids) == ["100", "101", "102", "103", "104"]
|
||||
|
||||
|
||||
def test_cian_serp_photo_urls():
|
||||
"""photo_urls — полные URL из offer.photos[].fullUrl."""
|
||||
offer = _make_full_offer(cian_id=999)
|
||||
html = _make_cian_serp_html([offer])
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
|
||||
lot = lots[0]
|
||||
assert len(lot.photo_urls) == 1
|
||||
assert "cdn-cian.ru" in lot.photo_urls[0]
|
||||
assert "999-1.jpg" in lot.photo_urls[0]
|
||||
|
||||
|
||||
def test_cian_serp_no_coords_does_not_crash():
|
||||
"""Если координаты отсутствуют — lot.lat/lon = None, не краш."""
|
||||
offer = _make_full_offer()
|
||||
del offer["geo"]["coordinates"]
|
||||
html = _make_cian_serp_html([offer])
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
|
||||
assert len(lots) == 1
|
||||
assert lots[0].lat is None
|
||||
assert lots[0].lon is None
|
||||
|
||||
|
||||
# ── Address formatter tests ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_format_address_standard():
|
||||
"""Стандартный geo.address[]."""
|
||||
parts = [
|
||||
{"type": "location", "name": "Свердловская область"},
|
||||
{"type": "city", "name": "Екатеринбург"},
|
||||
{"type": "district", "name": "Чкаловский"},
|
||||
{"type": "street", "name": "улица Постовского"},
|
||||
{"type": "house", "name": "17А"},
|
||||
{"type": "metro", "name": "Геологическая"},
|
||||
]
|
||||
result = _format_address(parts)
|
||||
# location и metro пропускаем
|
||||
assert "Свердловская" not in result
|
||||
assert "Геологическая" not in result
|
||||
assert "Екатеринбург" in result
|
||||
assert "17А" in result
|
||||
|
||||
|
||||
def test_format_address_empty():
|
||||
"""Пустой address[] → fallback string."""
|
||||
assert _format_address([]) == "Екатеринбург (Cian)"
|
||||
Loading…
Add table
Reference in a new issue