gendesign/tradein-mvp/backend/app/services/scrapers/cian.py
bot-backend e63bdd9c65 feat(tradein): dedicated Cian mobile proxy — separate egress from Avito (#cian-proxy)
Cian и Avito делили один мобильный IP (scraper_proxy_url) → конкуренция за egress →
взаимные таймауты/баны при параллельных прогонах. Вводит settings.cian_proxy_url
(CIAN_PROXY_URL, fallback на scraper_proxy_url) + cian_proxy_rotate_url; все 5
Cian-прокси-точек переключены. Avito/прочие не затронуты.
2026-05-31 22:27:30 +03:00

555 lines
26 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Cian.ru scraper — state-based SERP parser (137 fields per offer).
Стратегия (Stage 3 рефактор):
- Cian React micro-frontends хранят state в window._cianConfig['frontend-serp'].push().
- URL pattern: https://ekb.cian.ru/cat.php?deal_type=sale&offer_type=flat&engine_version=2
ekb.cian.ru — city-specific subdomain для ЕКБ (per Schema_Cian_SERP_Inventory sec 13).
ВАЖНО: Циан блокирует httpx по TLS fingerprint → "Обнаружен подозрительный трафик" (403).
Используем curl_cffi с impersonate='chrome120' — это libcurl-impersonate под капотом,
который воспроизводит TLS ClientHello как настоящий Chrome.
НЕ используем anchor jitter: offer.geo.coordinates.{lat,lng} — точные координаты
прямо из SERP state. Jitter запрещён per implementation plan.
"""
from __future__ import annotations
import hashlib
import logging
from datetime import UTC, datetime
from typing import Any
from urllib.parse import urlencode
import sentry_sdk
from curl_cffi.requests import AsyncSession
from app.core.config import settings
from app.services.scraper_settings import get_scraper_delay
from app.services.scrapers.base import BaseScraper, ScrapedLot
from app.services.scrapers.cian_state_parser import extract_state
from app.services.scrapers.repair_state_normalizer import (
infer_repair_state_from_text,
normalize_repair_state,
)
logger = logging.getLogger(__name__)
# Регион 4743 = Свердловская область (Cian internal region ID)
CIAN_EKB_REGION_ID = 4743
# SERP MFE name и state key (per Schema sec 1.2)
_MFE_SERP = "frontend-serp"
_STATE_KEY = "initialState"
class CianScraper(BaseScraper):
"""Cian SERP scraper. Использует curl_cffi для обхода TLS fingerprint.
Извлекает 137 полей на offer из Redux initialState через cian_state_parser.
Координаты точные — НЕТ anchor jitter (offer.geo.coordinates).
"""
name = "cian"
# ekb.cian.ru — city-specific subdomain для ЕКБ (per Schema sec 13, closed Q4)
base_url = "https://ekb.cian.ru"
# Класс-дефолт; реальное значение загружается из scraper_settings при создании экземпляра.
request_delay_sec = 5.0 # консервативно: Cian менее агрессивен чем Avito, но 5s безопасно
def __init__(self) -> None:
super().__init__()
self.request_delay_sec = get_scraper_delay(self.name)
self._cffi: AsyncSession | None = None
async def __aenter__(self) -> CianScraper:
await super().__aenter__()
# curl_cffi session с Chrome 120 TLS fingerprint — КРИТИЧЕСКИ ВАЖНО для Cian.
# proxies: mobile-proxy egress (#806) — аналогично Avito (#623). Без прокси
# datacenter-IP блокируется Cian (403/captcha) несмотря на валидные cookies.
# Пусто (SCRAPER_PROXY_URL/AVITO_PROXY_URL не заданы) → прямое подключение (dev).
_proxy_url = settings.cian_proxy_url
_proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
self._cffi = AsyncSession(
impersonate="chrome120",
timeout=30,
proxies=_proxies,
headers={
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
"Cache-Control": "max-age=0",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Upgrade-Insecure-Requests": "1",
},
)
# Warm-up: посетить homepage чтобы получить сессионные cookies (_CIAN_GK/_yasc)
# и не выглядеть как наглый бот (первый запрос сразу на /cat.php подозрителен).
# Failure не критична — SERP-запросы продолжатся в любом случае.
try:
_warm = await self._cffi.get(f"{self.base_url}/")
logger.info(
"cian warm-up GET / → HTTP %d (cookies=%d)",
_warm.status_code,
len(list(self._cffi.cookies.jar)),
)
# Последующие SERP-запросы выглядят как in-site navigation (Referer = /)
self._cffi.headers.update(
{
"Referer": f"{self.base_url}/",
"Sec-Fetch-Site": "same-origin",
}
)
except Exception:
logger.warning("cian warm-up GET / failed — continuing without seed cookies")
return self
async def __aexit__(self, *args: Any) -> None:
if self._cffi is not None:
await self._cffi.close()
await super().__aexit__(*args)
async def fetch_around(
self,
lat: float,
lon: float,
radius_m: int = 1000,
rooms: tuple[int, ...] | None = None,
page: int = 1,
) -> list[ScrapedLot]:
"""Найти объявления Циан по ЕКБ (lat/lon для reference; Cian не поддерживает bbox).
Cian SERP отдаёт весь ЕКБ — фильтрация по радиусу происходит в postgres
через ST_DWithin после сохранения (точные coords из state).
rooms — список из 1,2,3,4 (Cian's room codes). Если None — все.
page — страница выдачи (Cian ~28 объявлений на страницу).
"""
url = self._build_url(rooms, page)
try:
assert self._cffi is not None
response = await self._cffi.get(url)
except Exception:
logger.exception("cian curl_cffi fetch failed for url=%s", url)
return []
if response.status_code != 200:
logger.warning("cian returned HTTP %d for url=%s", response.status_code, url)
return []
lots = self._parse_serp_html(response.text)
logger.info(
"cian: %d lots fetched rooms=%s page=%d url=%s",
len(lots),
rooms,
page,
url,
)
await self.sleep_between_requests()
return lots
async def fetch_around_multi_room(
self,
lat: float,
lon: float,
radius_m: int = 1000,
pages: int = 8,
) -> list[ScrapedLot]:
"""Скрейп Циан по 1к/2к/3к/4+ × N страниц — расширяет выборку.
4 комнаты × N страниц × ~28 = до ~330+ лотов до дедупликации.
"""
seen: dict[str, ScrapedLot] = {}
for rooms in ((1,), (2,), (3,), (4,)):
for page in range(1, pages + 1):
try:
lots = await self.fetch_around(lat, lon, radius_m, rooms=rooms, page=page)
except Exception:
logger.exception("cian multi-room fetch failed rooms=%s page=%d", rooms, page)
continue
if not lots:
break # пустая страница → дальше смысла нет
for lot in lots:
key = lot.source_id or lot.source_url
if key and key not in seen:
seen[key] = lot
logger.info(
"cian multi-room: %d unique lots (lat=%.4f lon=%.4f radius=%dm)",
len(seen),
lat,
lon,
radius_m,
)
return list(seen.values())
def _build_url(
self,
rooms: tuple[int, ...] | None = None,
page: int = 1,
) -> str:
"""URL для Cian каталога вторички ЕКБ.
Используем ekb.cian.ru (city-specific subdomain).
Регион задаётся через region= param как fallback для reliability.
"""
params: list[tuple[str, Any]] = [
("deal_type", "sale"),
("engine_version", "2"),
("offer_type", "flat"),
("region", CIAN_EKB_REGION_ID),
("sort", "creation_date_desc"),
]
if rooms:
for r in rooms:
params.append((f"room{r}", "1"))
if page > 1:
params.append(("p", page))
return f"{self.base_url}/cat.php?{urlencode(params)}"
def _parse_serp_html(self, html: str) -> list[ScrapedLot]:
"""Извлечь offers из Cian Redux state.
Использует cian_state_parser.extract_state() — общая утилита Stage 2.
Возвращает пустой список если state не найден.
"""
state = extract_state(html, mfe=_MFE_SERP, key=_STATE_KEY)
if state is None:
logger.warning(
"cian SERP state extraction failed (mfe=%s key=%s) — "
"возможно Cian изменил структуру или вернул captcha",
_MFE_SERP,
_STATE_KEY,
)
return []
offers_data: list[dict[str, Any]] = state.get("results", {}).get("offers", [])
if not offers_data:
logger.warning(
"cian state found but results.offers пуст (totalOffers=%s)",
state.get("results", {}).get("totalOffers", "?"),
)
return []
logger.info(
"cian SERP state ok: %d offers (totalOffers=%s)",
len(offers_data),
state.get("results", {}).get("totalOffers", "?"),
)
lots: list[ScrapedLot] = []
for offer in offers_data:
lot = self._offer_to_lot(offer)
if lot is not None:
lots.append(lot)
raw_count = len(offers_data)
saved_count = len(lots)
# Охрана от silent-failure: если получили offers, но ни один не прошёл парсинг
# — likely schema regression (как descriptionMinhash str→list[int] 2026-05-23).
if raw_count > 0 and saved_count == 0:
logger.error(
"cian SERP: 0/%d offers прошли _offer_to_lot — возможна schema regression",
raw_count,
)
try:
if settings.glitchtip_dsn:
sentry_sdk.capture_message(
f"cian SERP: {raw_count}/{raw_count} offers failed _offer_to_lot"
" — possible schema regression",
level="error",
)
except Exception:
pass # sentry_sdk not initialised in dev
return lots
def _offer_to_lot(self, offer: dict[str, Any]) -> ScrapedLot | None:
"""Парсинг одного Cian offer (137 fields) → ScrapedLot.
Маппинг полей per Schema_Cian_SERP_Inventory sec 3, sec 8, sec 11.
Без anchor jitter: координаты из offer.geo.coordinates (точные).
"""
try:
# ── Identity ─────────────────────────────────────────────────────
offer_id = offer.get("cianId") or offer.get("id")
if not offer_id:
logger.debug("cian offer пропущен: нет cianId/id")
return None
source_id = str(offer_id)
url = offer.get("fullUrl")
if not url:
url = f"{self.base_url}/sale/flat/{offer_id}/"
# ── Цена ─────────────────────────────────────────────────────────
bargain: dict[str, Any] = offer.get("bargainTerms") or {}
price = bargain.get("priceRur") or bargain.get("price")
if not price:
logger.debug("cian offer %s пропущен: нет цены", offer_id)
return None
try:
price_rub = int(price)
except (TypeError, ValueError):
logger.debug("cian offer %s: не удалось распарсить цену %r", offer_id, price)
return None
if price_rub <= 0:
return None
# ── Параметры квартиры ───────────────────────────────────────────
rooms: int | None = offer.get("roomsCount")
# Студия: flatType == 'studio' → 0 комнат
if offer.get("flatType") == "studio" and rooms is None:
rooms = 0
area_raw = offer.get("totalArea")
area_m2: float | None = None
if area_raw is not None:
try:
area_m2 = float(area_raw)
except (TypeError, ValueError):
pass
living_area_raw = offer.get("livingArea")
living_area_m2: float | None = None
if living_area_raw is not None:
try:
living_area_m2 = float(living_area_raw)
except (TypeError, ValueError):
pass
# kitchen_area_m2 сохраняем в raw_payload (поле не в ScrapedLot Stage 2)
kitchen_area_raw = offer.get("kitchenArea")
floor: int | None = offer.get("floorNumber")
# ── Здание ───────────────────────────────────────────────────────
building: dict[str, Any] = offer.get("building") or {}
total_floors: int | None = building.get("floorsCount")
year_built: int | None = building.get("buildYear")
house_type: str | None = building.get("materialType")
# Newbuilding deadline year — если нет buildYear
if year_built is None:
deadline = building.get("deadline") or {}
year_built = deadline.get("year")
# ── Кадастр ─────────────────────────────────────────────────────
# cadastralNumber — кадастр КВАРТИРЫ
cadastral_number: str | None = offer.get("cadastralNumber")
# buildingCadastralNumber — кадастр ДОМА
building_cadastral_number: str | None = offer.get("buildingCadastralNumber")
# ── Геолокация (точные координаты — NO jitter) ───────────────────
geo: dict[str, Any] = offer.get("geo") or {}
coords: dict[str, Any] = geo.get("coordinates") or {}
lat: float | None = coords.get("lat")
lon: float | None = coords.get("lng")
# Если нет coords → сохраняем без lat/lon (geocode-missing обработает позже)
if lat is None or lon is None:
logger.debug(
"cian offer %s: нет точных координат в state (geo=%s)",
offer_id,
coords,
)
lat = lon = None
# ── Адрес из geo.address[] ───────────────────────────────────────
address_parts = geo.get("address") or []
address = _format_address(address_parts)
# ── Metro stations ───────────────────────────────────────────────
undergrounds: list[dict[str, Any]] = geo.get("undergrounds") or []
metro_stations = [
{
"name": u.get("name"),
"time": u.get("time"),
"mode": u.get("transportType"),
"line_color": u.get("lineColor"),
"line_id": u.get("lineId"),
"is_default": u.get("isDefault", False),
}
for u in undergrounds
if u.get("name")
]
# ── Newbuilding link ─────────────────────────────────────────────
newbuilding: dict[str, Any] = offer.get("newbuilding") or {}
nb_id = newbuilding.get("id")
# id == 0 — sentinel для вторички (per Schema sec 20.5)
house_source: str | None = None
house_ext_id: str | None = None
listing_segment: str | None = None
if nb_id and int(nb_id) > 0:
house_source = "cian_newbuilding"
house_ext_id = str(nb_id)
listing_segment = "novostroyki"
else:
house_source = "cian"
listing_segment = "vtorichka"
# ── Дополнительные характеристики ────────────────────────────────
balconies_count: int | None = offer.get("balconiesCount")
loggias_count: int | None = offer.get("loggiasCount")
bedrooms_count: int | None = offer.get("bedroomsCount")
has_balcony: bool | None = None
if balconies_count is not None:
has_balcony = balconies_count > 0
elif loggias_count is not None:
has_balcony = loggias_count > 0
# Мебель и repair_state
has_furniture: bool | None = offer.get("hasFurniture") or offer.get("isSoldFurnished")
# decoration — для новостроек (отделка); repair_state — для вторички
# Cian SERP: offer.decoration содержит raw значения (without/cosmetic/euro/design/...)
# Нормализуем до enum needs_repair/standard/good/excellent при инgesте.
repair_state: str | None = normalize_repair_state(offer.get("decoration"))
# ── Продавец ─────────────────────────────────────────────────────
phones: list[dict[str, Any]] = offer.get("phones") or []
is_homeowner: bool | None = offer.get("isByHomeowner")
is_pro_seller: bool | None = offer.get("isPro")
# ── Сделка ───────────────────────────────────────────────────────
sale_type: str | None = bargain.get("saleType")
bargain_allowed: bool | None = bargain.get("bargainAllowed")
# ── Description + minhash ─────────────────────────────────────────
description: str | None = offer.get("description")
# Cian предоставляет готовый minhash для dedup и cross-source matching.
# С ~2026-05-23 Cian изменил тип descriptionMinhash: str → list[int].
# Нормализуем оба варианта; НЕ sorted() — порядок band-ов значим для LSH.
_raw_minhash = offer.get("descriptionMinhash")
if isinstance(_raw_minhash, list):
description_minhash: str | None = ",".join(str(x) for x in _raw_minhash) or None
elif isinstance(_raw_minhash, str):
description_minhash = _raw_minhash or None
else:
description_minhash = None
# Если Cian не дал minhash — вычисляем простой SHA1 из description
if not description_minhash and description:
description_minhash = hashlib.sha1(
description.lower().encode("utf-8", errors="replace")
).hexdigest()[:32]
# Fallback: repair_state из описания, если decoration отсутствует (#622)
if repair_state is None and description:
repair_state = infer_repair_state_from_text(description)
# ── Фото ─────────────────────────────────────────────────────────
photo_urls: list[str] = [
p["fullUrl"]
for p in (offer.get("photos") or [])
if isinstance(p, dict) and p.get("fullUrl")
]
# ── Дата публикации ───────────────────────────────────────────────
added_ts = offer.get("addedTimestamp")
listing_date = None
if added_ts:
try:
listing_date = datetime.fromtimestamp(int(added_ts), tz=UTC).date()
except (TypeError, ValueError, OSError):
pass
# ── Raw payload (полный offer для enrichment в будущем) ───────────
raw_payload: dict[str, Any] = {
"cian_id": offer_id,
"flat_type": offer.get("flatType"),
"is_apartments": offer.get("isApartments"),
"offer_type": offer.get("offerType"),
"category": offer.get("category"),
"has_furniture": has_furniture,
"kitchen_area_m2": kitchen_area_raw, # не в ScrapedLot — сохраняем здесь
"mortgage_allowed": bargain.get("mortgageAllowed"),
"sale_type": sale_type,
"newbuilding_name": newbuilding.get("name"),
"is_from_developer": (
offer.get("fromDeveloper") or newbuilding.get("isFromDeveloper")
),
"builders_ids": offer.get("buildersIds"),
"is_rosreestr_checked": offer.get("isRosreestrChecked"),
"is_layout_approved": offer.get("isLayoutApproved"),
"user_id": offer.get("userId"),
"published_user_id": offer.get("publishedUserId"),
"is_by_commercial_owner": offer.get("isByCommercialOwner"),
"is_cian_partner": offer.get("isCianPartner"),
"description_words_highlighted": offer.get("descriptionWordsHighlighted"),
"building_parking": building.get("parking"),
"building_total_area": building.get("totalArea"),
}
return ScrapedLot(
source="cian",
source_url=url,
source_id=source_id,
address=address,
lat=lat,
lon=lon,
rooms=rooms,
area_m2=area_m2,
floor=floor,
total_floors=total_floors,
year_built=year_built,
house_type=house_type,
repair_state=repair_state,
has_balcony=has_balcony,
kadastr_num=cadastral_number,
house_source=house_source,
house_ext_id=house_ext_id,
listing_segment=listing_segment,
price_rub=price_rub,
price_per_m2=None, # compute_price_per_m2() вычислит
listing_date=listing_date,
photo_urls=photo_urls,
raw_payload=raw_payload,
# Cian-specific (Stage 2 fields)
living_area_m2=living_area_m2,
bedrooms_count=bedrooms_count,
balconies_count=balconies_count,
loggias_count=loggias_count,
description_minhash=description_minhash,
cadastral_number=cadastral_number,
building_cadastral_number=building_cadastral_number,
phones=phones,
is_homeowner=is_homeowner,
is_pro_seller=is_pro_seller,
bargain_allowed=bargain_allowed,
sale_type=sale_type,
metro_stations=metro_stations,
)
except Exception:
_oid = offer.get("cianId") or offer.get("id")
logger.exception("cian _offer_to_lot failed for offer_id=%s", _oid)
return None
# ── Address formatter ────────────────────────────────────────────────────────
def _format_address(address_parts: list[dict[str, Any]]) -> str:
"""Сформировать читаемый адрес из geo.address[] Cian.
Пропускаем location (страна/регион) и metro — берём раион/улицу/дом.
Пример: [location=Москва, raion=Пресненский, street=..., house=1С7, metro=Москва-Сити]
'Пресненский, улица ..., 1С7'
"""
if not address_parts:
return "Екатеринбург (Cian)"
skip_types = {"location", "metro"}
parts: list[str] = []
for part in address_parts:
if not isinstance(part, dict):
continue
ptype = part.get("type", "")
if ptype in skip_types:
continue
name = (part.get("fullName") or part.get("name") or "").strip()
if name:
parts.append(name)
return ", ".join(parts) if parts else "Екатеринбург (Cian)"