"""Cian.ru scraper — state-based SERP parser (137 fields per offer). Стратегия (Stage 3 рефактор): - Cian React micro-frontends хранят state в window._cianConfig['frontend-serp'].push(). - URL pattern: https://ekb.cian.ru/cat.php?deal_type=sale&offer_type=flat&engine_version=2 ekb.cian.ru — city-specific subdomain для ЕКБ (per Schema_Cian_SERP_Inventory sec 13). ВАЖНО: Циан блокирует httpx по TLS fingerprint → "Обнаружен подозрительный трафик" (403). Используем curl_cffi с impersonate='chrome120' — это libcurl-impersonate под капотом, который воспроизводит TLS ClientHello как настоящий Chrome. НЕ используем anchor jitter: offer.geo.coordinates.{lat,lng} — точные координаты прямо из SERP state. Jitter запрещён per implementation plan. """ from __future__ import annotations import hashlib import logging from datetime import UTC, datetime from typing import Any from urllib.parse import urlencode from curl_cffi.requests import AsyncSession from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper, ScrapedLot from app.services.scrapers.cian_state_parser import extract_state from app.services.scrapers.repair_state_normalizer import ( infer_repair_state_from_text, normalize_repair_state, ) logger = logging.getLogger(__name__) # Регион 4743 = Свердловская область (Cian internal region ID) CIAN_EKB_REGION_ID = 4743 # SERP MFE name и state key (per Schema sec 1.2) _MFE_SERP = "frontend-serp" _STATE_KEY = "initialState" class CianScraper(BaseScraper): """Cian SERP scraper. Использует curl_cffi для обхода TLS fingerprint. Извлекает 137 полей на offer из Redux initialState через cian_state_parser. Координаты точные — НЕТ anchor jitter (offer.geo.coordinates). """ name = "cian" # ekb.cian.ru — city-specific subdomain для ЕКБ (per Schema sec 13, closed Q4) base_url = "https://ekb.cian.ru" # Класс-дефолт; реальное значение загружается из scraper_settings при создании экземпляра. request_delay_sec = 5.0 # консервативно: Cian менее агрессивен чем Avito, но 5s безопасно def __init__(self) -> None: super().__init__() self.request_delay_sec = get_scraper_delay(self.name) self._cffi: AsyncSession | None = None async def __aenter__(self) -> CianScraper: await super().__aenter__() # curl_cffi session с Chrome 120 TLS fingerprint — КРИТИЧЕСКИ ВАЖНО для Cian self._cffi = AsyncSession( impersonate="chrome120", timeout=30, headers={ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", "Cache-Control": "max-age=0", "Sec-Fetch-Dest": "document", "Sec-Fetch-Mode": "navigate", "Sec-Fetch-Site": "none", "Sec-Fetch-User": "?1", "Upgrade-Insecure-Requests": "1", }, ) return self async def __aexit__(self, *args: Any) -> None: if self._cffi is not None: await self._cffi.close() await super().__aexit__(*args) async def fetch_around( self, lat: float, lon: float, radius_m: int = 1000, rooms: tuple[int, ...] | None = None, page: int = 1, ) -> list[ScrapedLot]: """Найти объявления Циан по ЕКБ (lat/lon для reference; Cian не поддерживает bbox). Cian SERP отдаёт весь ЕКБ — фильтрация по радиусу происходит в postgres через ST_DWithin после сохранения (точные coords из state). rooms — список из 1,2,3,4 (Cian's room codes). Если None — все. page — страница выдачи (Cian ~28 объявлений на страницу). """ url = self._build_url(rooms, page) try: assert self._cffi is not None response = await self._cffi.get(url) except Exception: logger.exception("cian curl_cffi fetch failed for url=%s", url) return [] if response.status_code != 200: logger.warning("cian returned HTTP %d for url=%s", response.status_code, url) return [] lots = self._parse_serp_html(response.text) logger.info( "cian: %d lots fetched rooms=%s page=%d url=%s", len(lots), rooms, page, url, ) await self.sleep_between_requests() return lots async def fetch_around_multi_room( self, lat: float, lon: float, radius_m: int = 1000, pages: int = 8, ) -> list[ScrapedLot]: """Скрейп Циан по 1к/2к/3к/4+ × N страниц — расширяет выборку. 4 комнаты × N страниц × ~28 = до ~330+ лотов до дедупликации. """ seen: dict[str, ScrapedLot] = {} for rooms in ((1,), (2,), (3,), (4,)): for page in range(1, pages + 1): try: lots = await self.fetch_around(lat, lon, radius_m, rooms=rooms, page=page) except Exception: logger.exception( "cian multi-room fetch failed rooms=%s page=%d", rooms, page ) continue if not lots: break # пустая страница → дальше смысла нет for lot in lots: key = lot.source_id or lot.source_url if key and key not in seen: seen[key] = lot logger.info( "cian multi-room: %d unique lots (lat=%.4f lon=%.4f radius=%dm)", len(seen), lat, lon, radius_m, ) return list(seen.values()) def _build_url( self, rooms: tuple[int, ...] | None = None, page: int = 1, ) -> str: """URL для Cian каталога вторички ЕКБ. Используем ekb.cian.ru (city-specific subdomain). Регион задаётся через region= param как fallback для reliability. """ params: list[tuple[str, Any]] = [ ("deal_type", "sale"), ("engine_version", "2"), ("offer_type", "flat"), ("region", CIAN_EKB_REGION_ID), ("sort", "creation_date_desc"), ] if rooms: for r in rooms: params.append((f"room{r}", "1")) if page > 1: params.append(("p", page)) return f"{self.base_url}/cat.php?{urlencode(params)}" def _parse_serp_html(self, html: str) -> list[ScrapedLot]: """Извлечь offers из Cian Redux state. Использует cian_state_parser.extract_state() — общая утилита Stage 2. Возвращает пустой список если state не найден. """ state = extract_state(html, mfe=_MFE_SERP, key=_STATE_KEY) if state is None: logger.warning( "cian SERP state extraction failed (mfe=%s key=%s) — " "возможно Cian изменил структуру или вернул captcha", _MFE_SERP, _STATE_KEY, ) return [] offers_data: list[dict[str, Any]] = ( state.get("results", {}).get("offers", []) ) if not offers_data: logger.warning( "cian state found but results.offers пуст (totalOffers=%s)", state.get("results", {}).get("totalOffers", "?"), ) return [] logger.info( "cian SERP state ok: %d offers (totalOffers=%s)", len(offers_data), state.get("results", {}).get("totalOffers", "?"), ) lots: list[ScrapedLot] = [] for offer in offers_data: lot = self._offer_to_lot(offer) if lot is not None: lots.append(lot) return lots def _offer_to_lot(self, offer: dict[str, Any]) -> ScrapedLot | None: """Парсинг одного Cian offer (137 fields) → ScrapedLot. Маппинг полей per Schema_Cian_SERP_Inventory sec 3, sec 8, sec 11. Без anchor jitter: координаты из offer.geo.coordinates (точные). """ try: # ── Identity ───────────────────────────────────────────────────── offer_id = offer.get("cianId") or offer.get("id") if not offer_id: logger.debug("cian offer пропущен: нет cianId/id") return None source_id = str(offer_id) url = offer.get("fullUrl") if not url: url = f"{self.base_url}/sale/flat/{offer_id}/" # ── Цена ───────────────────────────────────────────────────────── bargain: dict[str, Any] = offer.get("bargainTerms") or {} price = bargain.get("priceRur") or bargain.get("price") if not price: logger.debug("cian offer %s пропущен: нет цены", offer_id) return None try: price_rub = int(price) except (TypeError, ValueError): logger.debug("cian offer %s: не удалось распарсить цену %r", offer_id, price) return None if price_rub <= 0: return None # ── Параметры квартиры ─────────────────────────────────────────── rooms: int | None = offer.get("roomsCount") # Студия: flatType == 'studio' → 0 комнат if offer.get("flatType") == "studio" and rooms is None: rooms = 0 area_raw = offer.get("totalArea") area_m2: float | None = None if area_raw is not None: try: area_m2 = float(area_raw) except (TypeError, ValueError): pass living_area_raw = offer.get("livingArea") living_area_m2: float | None = None if living_area_raw is not None: try: living_area_m2 = float(living_area_raw) except (TypeError, ValueError): pass # kitchen_area_m2 сохраняем в raw_payload (поле не в ScrapedLot Stage 2) kitchen_area_raw = offer.get("kitchenArea") floor: int | None = offer.get("floorNumber") # ── Здание ─────────────────────────────────────────────────────── building: dict[str, Any] = offer.get("building") or {} total_floors: int | None = building.get("floorsCount") year_built: int | None = building.get("buildYear") house_type: str | None = building.get("materialType") # Newbuilding deadline year — если нет buildYear if year_built is None: deadline = building.get("deadline") or {} year_built = deadline.get("year") # ── Кадастр ───────────────────────────────────────────────────── # cadastralNumber — кадастр КВАРТИРЫ cadastral_number: str | None = offer.get("cadastralNumber") # buildingCadastralNumber — кадастр ДОМА building_cadastral_number: str | None = offer.get("buildingCadastralNumber") # ── Геолокация (точные координаты — NO jitter) ─────────────────── geo: dict[str, Any] = offer.get("geo") or {} coords: dict[str, Any] = geo.get("coordinates") or {} lat: float | None = coords.get("lat") lon: float | None = coords.get("lng") # Если нет coords → сохраняем без lat/lon (geocode-missing обработает позже) if lat is None or lon is None: logger.debug( "cian offer %s: нет точных координат в state (geo=%s)", offer_id, coords, ) lat = lon = None # ── Адрес из geo.address[] ─────────────────────────────────────── address_parts = geo.get("address") or [] address = _format_address(address_parts) # ── Metro stations ─────────────────────────────────────────────── undergrounds: list[dict[str, Any]] = geo.get("undergrounds") or [] metro_stations = [ { "name": u.get("name"), "time": u.get("time"), "mode": u.get("transportType"), "line_color": u.get("lineColor"), "line_id": u.get("lineId"), "is_default": u.get("isDefault", False), } for u in undergrounds if u.get("name") ] # ── Newbuilding link ───────────────────────────────────────────── newbuilding: dict[str, Any] = offer.get("newbuilding") or {} nb_id = newbuilding.get("id") # id == 0 — sentinel для вторички (per Schema sec 20.5) house_source: str | None = None house_ext_id: str | None = None listing_segment: str | None = None if nb_id and int(nb_id) > 0: house_source = "cian_newbuilding" house_ext_id = str(nb_id) listing_segment = "novostroyki" else: house_source = "cian" listing_segment = "vtorichka" # ── Дополнительные характеристики ──────────────────────────────── balconies_count: int | None = offer.get("balconiesCount") loggias_count: int | None = offer.get("loggiasCount") bedrooms_count: int | None = offer.get("bedroomsCount") has_balcony: bool | None = None if balconies_count is not None: has_balcony = balconies_count > 0 elif loggias_count is not None: has_balcony = loggias_count > 0 # Мебель и repair_state has_furniture: bool | None = offer.get("hasFurniture") or offer.get("isSoldFurnished") # decoration — для новостроек (отделка); repair_state — для вторички # Cian SERP: offer.decoration содержит raw значения (without/cosmetic/euro/design/...) # Нормализуем до enum needs_repair/standard/good/excellent при инgesте. repair_state: str | None = normalize_repair_state(offer.get("decoration")) # ── Продавец ───────────────────────────────────────────────────── phones: list[dict[str, Any]] = offer.get("phones") or [] is_homeowner: bool | None = offer.get("isByHomeowner") is_pro_seller: bool | None = offer.get("isPro") # ── Сделка ─────────────────────────────────────────────────────── sale_type: str | None = bargain.get("saleType") bargain_allowed: bool | None = bargain.get("bargainAllowed") # ── Description + minhash ───────────────────────────────────────── description: str | None = offer.get("description") # Cian предоставляет готовый minhash для dedup и cross-source matching description_minhash: str | None = offer.get("descriptionMinhash") # Если Cian не дал minhash — вычисляем простой SHA1 из description if not description_minhash and description: description_minhash = hashlib.sha1( description.lower().encode("utf-8", errors="replace") ).hexdigest()[:32] # Fallback: repair_state из описания, если decoration отсутствует (#622) if repair_state is None and description: repair_state = infer_repair_state_from_text(description) # ── Фото ───────────────────────────────────────────────────────── photo_urls: list[str] = [ p["fullUrl"] for p in (offer.get("photos") or []) if isinstance(p, dict) and p.get("fullUrl") ] # ── Дата публикации ─────────────────────────────────────────────── added_ts = offer.get("addedTimestamp") listing_date = None if added_ts: try: listing_date = datetime.fromtimestamp(int(added_ts), tz=UTC).date() except (TypeError, ValueError, OSError): pass # ── Raw payload (полный offer для enrichment в будущем) ─────────── raw_payload: dict[str, Any] = { "cian_id": offer_id, "flat_type": offer.get("flatType"), "is_apartments": offer.get("isApartments"), "offer_type": offer.get("offerType"), "category": offer.get("category"), "has_furniture": has_furniture, "kitchen_area_m2": kitchen_area_raw, # не в ScrapedLot — сохраняем здесь "mortgage_allowed": bargain.get("mortgageAllowed"), "sale_type": sale_type, "newbuilding_name": newbuilding.get("name"), "is_from_developer": ( offer.get("fromDeveloper") or newbuilding.get("isFromDeveloper") ), "builders_ids": offer.get("buildersIds"), "is_rosreestr_checked": offer.get("isRosreestrChecked"), "is_layout_approved": offer.get("isLayoutApproved"), "user_id": offer.get("userId"), "published_user_id": offer.get("publishedUserId"), "is_by_commercial_owner": offer.get("isByCommercialOwner"), "is_cian_partner": offer.get("isCianPartner"), "description_words_highlighted": offer.get("descriptionWordsHighlighted"), "building_parking": building.get("parking"), "building_total_area": building.get("totalArea"), } return ScrapedLot( source="cian", source_url=url, source_id=source_id, address=address, lat=lat, lon=lon, rooms=rooms, area_m2=area_m2, floor=floor, total_floors=total_floors, year_built=year_built, house_type=house_type, repair_state=repair_state, has_balcony=has_balcony, kadastr_num=cadastral_number, house_source=house_source, house_ext_id=house_ext_id, listing_segment=listing_segment, price_rub=price_rub, price_per_m2=None, # compute_price_per_m2() вычислит listing_date=listing_date, photo_urls=photo_urls, raw_payload=raw_payload, # Cian-specific (Stage 2 fields) living_area_m2=living_area_m2, bedrooms_count=bedrooms_count, balconies_count=balconies_count, loggias_count=loggias_count, description_minhash=description_minhash, cadastral_number=cadastral_number, building_cadastral_number=building_cadastral_number, phones=phones, is_homeowner=is_homeowner, is_pro_seller=is_pro_seller, bargain_allowed=bargain_allowed, sale_type=sale_type, metro_stations=metro_stations, ) except Exception: _oid = offer.get("cianId") or offer.get("id") logger.exception("cian _offer_to_lot failed for offer_id=%s", _oid) return None # ── Address formatter ──────────────────────────────────────────────────────── def _format_address(address_parts: list[dict[str, Any]]) -> str: """Сформировать читаемый адрес из geo.address[] Cian. Пропускаем location (страна/регион) и metro — берём раион/улицу/дом. Пример: [location=Москва, raion=Пресненский, street=..., house=1С7, metro=Москва-Сити] → 'Пресненский, улица ..., 1С7' """ if not address_parts: return "Екатеринбург (Cian)" skip_types = {"location", "metro"} parts: list[str] = [] for part in address_parts: if not isinstance(part, dict): continue ptype = part.get("type", "") if ptype in skip_types: continue name = (part.get("fullName") or part.get("name") or "").strip() if name: parts.append(name) return ", ".join(parts) if parts else "Екатеринбург (Cian)"