496 lines
23 KiB
Python
496 lines
23 KiB
Python
"""Cian.ru scraper — state-based SERP parser (137 fields per offer).
|
||
|
||
Стратегия (Stage 3 рефактор):
|
||
- Cian React micro-frontends хранят state в window._cianConfig['frontend-serp'].push().
|
||
- URL pattern: https://ekb.cian.ru/cat.php?deal_type=sale&offer_type=flat&engine_version=2
|
||
ekb.cian.ru — city-specific subdomain для ЕКБ (per Schema_Cian_SERP_Inventory sec 13).
|
||
|
||
ВАЖНО: Циан блокирует httpx по TLS fingerprint → "Обнаружен подозрительный трафик" (403).
|
||
Используем curl_cffi с impersonate='chrome120' — это libcurl-impersonate под капотом,
|
||
который воспроизводит TLS ClientHello как настоящий Chrome.
|
||
|
||
НЕ используем anchor jitter: offer.geo.coordinates.{lat,lng} — точные координаты
|
||
прямо из SERP state. Jitter запрещён per implementation plan.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import logging
|
||
from datetime import UTC, datetime
|
||
from typing import Any
|
||
from urllib.parse import urlencode
|
||
|
||
from curl_cffi.requests import AsyncSession
|
||
|
||
from app.services.scraper_settings import get_scraper_delay
|
||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||
from app.services.scrapers.cian_state_parser import extract_state
|
||
from app.services.scrapers.repair_state_normalizer import (
|
||
infer_repair_state_from_text,
|
||
normalize_repair_state,
|
||
)
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# Регион 4743 = Свердловская область (Cian internal region ID)
|
||
CIAN_EKB_REGION_ID = 4743
|
||
|
||
# SERP MFE name и state key (per Schema sec 1.2)
|
||
_MFE_SERP = "frontend-serp"
|
||
_STATE_KEY = "initialState"
|
||
|
||
|
||
class CianScraper(BaseScraper):
|
||
"""Cian SERP scraper. Использует curl_cffi для обхода TLS fingerprint.
|
||
|
||
Извлекает 137 полей на offer из Redux initialState через cian_state_parser.
|
||
Координаты точные — НЕТ anchor jitter (offer.geo.coordinates).
|
||
"""
|
||
|
||
name = "cian"
|
||
# ekb.cian.ru — city-specific subdomain для ЕКБ (per Schema sec 13, closed Q4)
|
||
base_url = "https://ekb.cian.ru"
|
||
# Класс-дефолт; реальное значение загружается из scraper_settings при создании экземпляра.
|
||
request_delay_sec = 5.0 # консервативно: Cian менее агрессивен чем Avito, но 5s безопасно
|
||
|
||
def __init__(self) -> None:
|
||
super().__init__()
|
||
self.request_delay_sec = get_scraper_delay(self.name)
|
||
self._cffi: AsyncSession | None = None
|
||
|
||
async def __aenter__(self) -> CianScraper:
|
||
await super().__aenter__()
|
||
# curl_cffi session с Chrome 120 TLS fingerprint — КРИТИЧЕСКИ ВАЖНО для Cian
|
||
self._cffi = AsyncSession(
|
||
impersonate="chrome120",
|
||
timeout=30,
|
||
headers={
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
|
||
"Cache-Control": "max-age=0",
|
||
"Sec-Fetch-Dest": "document",
|
||
"Sec-Fetch-Mode": "navigate",
|
||
"Sec-Fetch-Site": "none",
|
||
"Sec-Fetch-User": "?1",
|
||
"Upgrade-Insecure-Requests": "1",
|
||
},
|
||
)
|
||
return self
|
||
|
||
async def __aexit__(self, *args: Any) -> None:
|
||
if self._cffi is not None:
|
||
await self._cffi.close()
|
||
await super().__aexit__(*args)
|
||
|
||
async def fetch_around(
|
||
self,
|
||
lat: float,
|
||
lon: float,
|
||
radius_m: int = 1000,
|
||
rooms: tuple[int, ...] | None = None,
|
||
page: int = 1,
|
||
) -> list[ScrapedLot]:
|
||
"""Найти объявления Циан по ЕКБ (lat/lon для reference; Cian не поддерживает bbox).
|
||
|
||
Cian SERP отдаёт весь ЕКБ — фильтрация по радиусу происходит в postgres
|
||
через ST_DWithin после сохранения (точные coords из state).
|
||
|
||
rooms — список из 1,2,3,4 (Cian's room codes). Если None — все.
|
||
page — страница выдачи (Cian ~28 объявлений на страницу).
|
||
"""
|
||
url = self._build_url(rooms, page)
|
||
try:
|
||
assert self._cffi is not None
|
||
response = await self._cffi.get(url)
|
||
except Exception:
|
||
logger.exception("cian curl_cffi fetch failed for url=%s", url)
|
||
return []
|
||
|
||
if response.status_code != 200:
|
||
logger.warning("cian returned HTTP %d for url=%s", response.status_code, url)
|
||
return []
|
||
|
||
lots = self._parse_serp_html(response.text)
|
||
logger.info(
|
||
"cian: %d lots fetched rooms=%s page=%d url=%s",
|
||
len(lots), rooms, page, url,
|
||
)
|
||
await self.sleep_between_requests()
|
||
return lots
|
||
|
||
async def fetch_around_multi_room(
|
||
self,
|
||
lat: float,
|
||
lon: float,
|
||
radius_m: int = 1000,
|
||
pages: int = 8,
|
||
) -> list[ScrapedLot]:
|
||
"""Скрейп Циан по 1к/2к/3к/4+ × N страниц — расширяет выборку.
|
||
|
||
4 комнаты × N страниц × ~28 = до ~330+ лотов до дедупликации.
|
||
"""
|
||
seen: dict[str, ScrapedLot] = {}
|
||
for rooms in ((1,), (2,), (3,), (4,)):
|
||
for page in range(1, pages + 1):
|
||
try:
|
||
lots = await self.fetch_around(lat, lon, radius_m, rooms=rooms, page=page)
|
||
except Exception:
|
||
logger.exception(
|
||
"cian multi-room fetch failed rooms=%s page=%d", rooms, page
|
||
)
|
||
continue
|
||
if not lots:
|
||
break # пустая страница → дальше смысла нет
|
||
for lot in lots:
|
||
key = lot.source_id or lot.source_url
|
||
if key and key not in seen:
|
||
seen[key] = lot
|
||
logger.info(
|
||
"cian multi-room: %d unique lots (lat=%.4f lon=%.4f radius=%dm)",
|
||
len(seen), lat, lon, radius_m,
|
||
)
|
||
return list(seen.values())
|
||
|
||
def _build_url(
|
||
self,
|
||
rooms: tuple[int, ...] | None = None,
|
||
page: int = 1,
|
||
) -> str:
|
||
"""URL для Cian каталога вторички ЕКБ.
|
||
|
||
Используем ekb.cian.ru (city-specific subdomain).
|
||
Регион задаётся через region= param как fallback для reliability.
|
||
"""
|
||
params: list[tuple[str, Any]] = [
|
||
("deal_type", "sale"),
|
||
("engine_version", "2"),
|
||
("offer_type", "flat"),
|
||
("region", CIAN_EKB_REGION_ID),
|
||
("sort", "creation_date_desc"),
|
||
]
|
||
if rooms:
|
||
for r in rooms:
|
||
params.append((f"room{r}", "1"))
|
||
if page > 1:
|
||
params.append(("p", page))
|
||
return f"{self.base_url}/cat.php?{urlencode(params)}"
|
||
|
||
def _parse_serp_html(self, html: str) -> list[ScrapedLot]:
|
||
"""Извлечь offers из Cian Redux state.
|
||
|
||
Использует cian_state_parser.extract_state() — общая утилита Stage 2.
|
||
Возвращает пустой список если state не найден.
|
||
"""
|
||
state = extract_state(html, mfe=_MFE_SERP, key=_STATE_KEY)
|
||
if state is None:
|
||
logger.warning(
|
||
"cian SERP state extraction failed (mfe=%s key=%s) — "
|
||
"возможно Cian изменил структуру или вернул captcha",
|
||
_MFE_SERP, _STATE_KEY,
|
||
)
|
||
return []
|
||
|
||
offers_data: list[dict[str, Any]] = (
|
||
state.get("results", {}).get("offers", [])
|
||
)
|
||
if not offers_data:
|
||
logger.warning(
|
||
"cian state found but results.offers пуст (totalOffers=%s)",
|
||
state.get("results", {}).get("totalOffers", "?"),
|
||
)
|
||
return []
|
||
|
||
logger.info(
|
||
"cian SERP state ok: %d offers (totalOffers=%s)",
|
||
len(offers_data),
|
||
state.get("results", {}).get("totalOffers", "?"),
|
||
)
|
||
|
||
lots: list[ScrapedLot] = []
|
||
for offer in offers_data:
|
||
lot = self._offer_to_lot(offer)
|
||
if lot is not None:
|
||
lots.append(lot)
|
||
return lots
|
||
|
||
def _offer_to_lot(self, offer: dict[str, Any]) -> ScrapedLot | None:
|
||
"""Парсинг одного Cian offer (137 fields) → ScrapedLot.
|
||
|
||
Маппинг полей per Schema_Cian_SERP_Inventory sec 3, sec 8, sec 11.
|
||
Без anchor jitter: координаты из offer.geo.coordinates (точные).
|
||
"""
|
||
try:
|
||
# ── Identity ─────────────────────────────────────────────────────
|
||
offer_id = offer.get("cianId") or offer.get("id")
|
||
if not offer_id:
|
||
logger.debug("cian offer пропущен: нет cianId/id")
|
||
return None
|
||
|
||
source_id = str(offer_id)
|
||
url = offer.get("fullUrl")
|
||
if not url:
|
||
url = f"{self.base_url}/sale/flat/{offer_id}/"
|
||
|
||
# ── Цена ─────────────────────────────────────────────────────────
|
||
bargain: dict[str, Any] = offer.get("bargainTerms") or {}
|
||
price = bargain.get("priceRur") or bargain.get("price")
|
||
if not price:
|
||
logger.debug("cian offer %s пропущен: нет цены", offer_id)
|
||
return None
|
||
try:
|
||
price_rub = int(price)
|
||
except (TypeError, ValueError):
|
||
logger.debug("cian offer %s: не удалось распарсить цену %r", offer_id, price)
|
||
return None
|
||
if price_rub <= 0:
|
||
return None
|
||
|
||
# ── Параметры квартиры ───────────────────────────────────────────
|
||
rooms: int | None = offer.get("roomsCount")
|
||
# Студия: flatType == 'studio' → 0 комнат
|
||
if offer.get("flatType") == "studio" and rooms is None:
|
||
rooms = 0
|
||
|
||
area_raw = offer.get("totalArea")
|
||
area_m2: float | None = None
|
||
if area_raw is not None:
|
||
try:
|
||
area_m2 = float(area_raw)
|
||
except (TypeError, ValueError):
|
||
pass
|
||
|
||
living_area_raw = offer.get("livingArea")
|
||
living_area_m2: float | None = None
|
||
if living_area_raw is not None:
|
||
try:
|
||
living_area_m2 = float(living_area_raw)
|
||
except (TypeError, ValueError):
|
||
pass
|
||
|
||
# kitchen_area_m2 сохраняем в raw_payload (поле не в ScrapedLot Stage 2)
|
||
kitchen_area_raw = offer.get("kitchenArea")
|
||
|
||
floor: int | None = offer.get("floorNumber")
|
||
|
||
# ── Здание ───────────────────────────────────────────────────────
|
||
building: dict[str, Any] = offer.get("building") or {}
|
||
total_floors: int | None = building.get("floorsCount")
|
||
year_built: int | None = building.get("buildYear")
|
||
house_type: str | None = building.get("materialType")
|
||
|
||
# Newbuilding deadline year — если нет buildYear
|
||
if year_built is None:
|
||
deadline = building.get("deadline") or {}
|
||
year_built = deadline.get("year")
|
||
|
||
# ── Кадастр ─────────────────────────────────────────────────────
|
||
# cadastralNumber — кадастр КВАРТИРЫ
|
||
cadastral_number: str | None = offer.get("cadastralNumber")
|
||
# buildingCadastralNumber — кадастр ДОМА
|
||
building_cadastral_number: str | None = offer.get("buildingCadastralNumber")
|
||
|
||
# ── Геолокация (точные координаты — NO jitter) ───────────────────
|
||
geo: dict[str, Any] = offer.get("geo") or {}
|
||
coords: dict[str, Any] = geo.get("coordinates") or {}
|
||
lat: float | None = coords.get("lat")
|
||
lon: float | None = coords.get("lng")
|
||
# Если нет coords → сохраняем без lat/lon (geocode-missing обработает позже)
|
||
if lat is None or lon is None:
|
||
logger.debug(
|
||
"cian offer %s: нет точных координат в state (geo=%s)",
|
||
offer_id, coords,
|
||
)
|
||
lat = lon = None
|
||
|
||
# ── Адрес из geo.address[] ───────────────────────────────────────
|
||
address_parts = geo.get("address") or []
|
||
address = _format_address(address_parts)
|
||
|
||
# ── Metro stations ───────────────────────────────────────────────
|
||
undergrounds: list[dict[str, Any]] = geo.get("undergrounds") or []
|
||
metro_stations = [
|
||
{
|
||
"name": u.get("name"),
|
||
"time": u.get("time"),
|
||
"mode": u.get("transportType"),
|
||
"line_color": u.get("lineColor"),
|
||
"line_id": u.get("lineId"),
|
||
"is_default": u.get("isDefault", False),
|
||
}
|
||
for u in undergrounds
|
||
if u.get("name")
|
||
]
|
||
|
||
# ── Newbuilding link ─────────────────────────────────────────────
|
||
newbuilding: dict[str, Any] = offer.get("newbuilding") or {}
|
||
nb_id = newbuilding.get("id")
|
||
# id == 0 — sentinel для вторички (per Schema sec 20.5)
|
||
house_source: str | None = None
|
||
house_ext_id: str | None = None
|
||
listing_segment: str | None = None
|
||
|
||
if nb_id and int(nb_id) > 0:
|
||
house_source = "cian_newbuilding"
|
||
house_ext_id = str(nb_id)
|
||
listing_segment = "novostroyki"
|
||
else:
|
||
house_source = "cian"
|
||
listing_segment = "vtorichka"
|
||
|
||
# ── Дополнительные характеристики ────────────────────────────────
|
||
balconies_count: int | None = offer.get("balconiesCount")
|
||
loggias_count: int | None = offer.get("loggiasCount")
|
||
bedrooms_count: int | None = offer.get("bedroomsCount")
|
||
|
||
has_balcony: bool | None = None
|
||
if balconies_count is not None:
|
||
has_balcony = balconies_count > 0
|
||
elif loggias_count is not None:
|
||
has_balcony = loggias_count > 0
|
||
|
||
# Мебель и repair_state
|
||
has_furniture: bool | None = offer.get("hasFurniture") or offer.get("isSoldFurnished")
|
||
# decoration — для новостроек (отделка); repair_state — для вторички
|
||
# Cian SERP: offer.decoration содержит raw значения (without/cosmetic/euro/design/...)
|
||
# Нормализуем до enum needs_repair/standard/good/excellent при инgesте.
|
||
repair_state: str | None = normalize_repair_state(offer.get("decoration"))
|
||
|
||
# ── Продавец ─────────────────────────────────────────────────────
|
||
phones: list[dict[str, Any]] = offer.get("phones") or []
|
||
is_homeowner: bool | None = offer.get("isByHomeowner")
|
||
is_pro_seller: bool | None = offer.get("isPro")
|
||
|
||
# ── Сделка ───────────────────────────────────────────────────────
|
||
sale_type: str | None = bargain.get("saleType")
|
||
bargain_allowed: bool | None = bargain.get("bargainAllowed")
|
||
|
||
# ── Description + minhash ─────────────────────────────────────────
|
||
description: str | None = offer.get("description")
|
||
# Cian предоставляет готовый minhash для dedup и cross-source matching
|
||
description_minhash: str | None = offer.get("descriptionMinhash")
|
||
# Если Cian не дал minhash — вычисляем простой SHA1 из description
|
||
if not description_minhash and description:
|
||
description_minhash = hashlib.sha1(
|
||
description.lower().encode("utf-8", errors="replace")
|
||
).hexdigest()[:32]
|
||
|
||
# Fallback: repair_state из описания, если decoration отсутствует (#622)
|
||
if repair_state is None and description:
|
||
repair_state = infer_repair_state_from_text(description)
|
||
|
||
# ── Фото ─────────────────────────────────────────────────────────
|
||
photo_urls: list[str] = [
|
||
p["fullUrl"]
|
||
for p in (offer.get("photos") or [])
|
||
if isinstance(p, dict) and p.get("fullUrl")
|
||
]
|
||
|
||
# ── Дата публикации ───────────────────────────────────────────────
|
||
added_ts = offer.get("addedTimestamp")
|
||
listing_date = None
|
||
if added_ts:
|
||
try:
|
||
listing_date = datetime.fromtimestamp(int(added_ts), tz=UTC).date()
|
||
except (TypeError, ValueError, OSError):
|
||
pass
|
||
|
||
# ── Raw payload (полный offer для enrichment в будущем) ───────────
|
||
raw_payload: dict[str, Any] = {
|
||
"cian_id": offer_id,
|
||
"flat_type": offer.get("flatType"),
|
||
"is_apartments": offer.get("isApartments"),
|
||
"offer_type": offer.get("offerType"),
|
||
"category": offer.get("category"),
|
||
"has_furniture": has_furniture,
|
||
"kitchen_area_m2": kitchen_area_raw, # не в ScrapedLot — сохраняем здесь
|
||
"mortgage_allowed": bargain.get("mortgageAllowed"),
|
||
"sale_type": sale_type,
|
||
"newbuilding_name": newbuilding.get("name"),
|
||
"is_from_developer": (
|
||
offer.get("fromDeveloper") or newbuilding.get("isFromDeveloper")
|
||
),
|
||
"builders_ids": offer.get("buildersIds"),
|
||
"is_rosreestr_checked": offer.get("isRosreestrChecked"),
|
||
"is_layout_approved": offer.get("isLayoutApproved"),
|
||
"user_id": offer.get("userId"),
|
||
"published_user_id": offer.get("publishedUserId"),
|
||
"is_by_commercial_owner": offer.get("isByCommercialOwner"),
|
||
"is_cian_partner": offer.get("isCianPartner"),
|
||
"description_words_highlighted": offer.get("descriptionWordsHighlighted"),
|
||
"building_parking": building.get("parking"),
|
||
"building_total_area": building.get("totalArea"),
|
||
}
|
||
|
||
return ScrapedLot(
|
||
source="cian",
|
||
source_url=url,
|
||
source_id=source_id,
|
||
address=address,
|
||
lat=lat,
|
||
lon=lon,
|
||
rooms=rooms,
|
||
area_m2=area_m2,
|
||
floor=floor,
|
||
total_floors=total_floors,
|
||
year_built=year_built,
|
||
house_type=house_type,
|
||
repair_state=repair_state,
|
||
has_balcony=has_balcony,
|
||
kadastr_num=cadastral_number,
|
||
house_source=house_source,
|
||
house_ext_id=house_ext_id,
|
||
listing_segment=listing_segment,
|
||
price_rub=price_rub,
|
||
price_per_m2=None, # compute_price_per_m2() вычислит
|
||
listing_date=listing_date,
|
||
photo_urls=photo_urls,
|
||
raw_payload=raw_payload,
|
||
# Cian-specific (Stage 2 fields)
|
||
living_area_m2=living_area_m2,
|
||
bedrooms_count=bedrooms_count,
|
||
balconies_count=balconies_count,
|
||
loggias_count=loggias_count,
|
||
description_minhash=description_minhash,
|
||
cadastral_number=cadastral_number,
|
||
building_cadastral_number=building_cadastral_number,
|
||
phones=phones,
|
||
is_homeowner=is_homeowner,
|
||
is_pro_seller=is_pro_seller,
|
||
bargain_allowed=bargain_allowed,
|
||
sale_type=sale_type,
|
||
metro_stations=metro_stations,
|
||
)
|
||
|
||
except Exception:
|
||
_oid = offer.get("cianId") or offer.get("id")
|
||
logger.exception("cian _offer_to_lot failed for offer_id=%s", _oid)
|
||
return None
|
||
|
||
|
||
# ── Address formatter ────────────────────────────────────────────────────────
|
||
|
||
|
||
def _format_address(address_parts: list[dict[str, Any]]) -> str:
|
||
"""Сформировать читаемый адрес из geo.address[] Cian.
|
||
|
||
Пропускаем location (страна/регион) и metro — берём раион/улицу/дом.
|
||
Пример: [location=Москва, raion=Пресненский, street=..., house=1С7, metro=Москва-Сити]
|
||
→ 'Пресненский, улица ..., 1С7'
|
||
"""
|
||
if not address_parts:
|
||
return "Екатеринбург (Cian)"
|
||
|
||
skip_types = {"location", "metro"}
|
||
parts: list[str] = []
|
||
for part in address_parts:
|
||
if not isinstance(part, dict):
|
||
continue
|
||
ptype = part.get("type", "")
|
||
if ptype in skip_types:
|
||
continue
|
||
name = (part.get("fullName") or part.get("name") or "").strip()
|
||
if name:
|
||
parts.append(name)
|
||
|
||
return ", ".join(parts) if parts else "Екатеринбург (Cian)"
|