From 4f0319297ee979bb317af072288d1f8f2c298b84 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 23 May 2026 12:52:04 +0000 Subject: [PATCH] =?UTF-8?q?feat(tradein):=20cian.py=20SERP=20refactor=20?= =?UTF-8?q?=E2=80=94=20state-based=20extraction=20(137=20fields,=20no=20ji?= =?UTF-8?q?tter)=20(#450)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/app/services/scrapers/cian.py | 671 +++++++++--------- .../backend/tests/test_cian_serp_scraper.py | 458 ++++++++++++ 2 files changed, 793 insertions(+), 336 deletions(-) create mode 100644 tradein-mvp/backend/tests/test_cian_serp_scraper.py diff --git a/tradein-mvp/backend/app/services/scrapers/cian.py b/tradein-mvp/backend/app/services/scrapers/cian.py index 97b2c1ec..8ad29df3 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian.py +++ b/tradein-mvp/backend/app/services/scrapers/cian.py @@ -1,51 +1,63 @@ -"""Cian.ru scraper — парсинг вторички вокруг точки. +"""Cian.ru scraper — state-based SERP parser (137 fields per offer). -Стратегия: -- Cian.ru Next.js приложение хранит state в `__NEXT_DATA__` JSON (script tag в HTML). -- URL pattern: https://ekaterinburg.cian.ru/cat.php?deal_type=sale&offer_type=flat&engine_version=2 +Стратегия (Stage 3 рефактор): +- Cian React micro-frontends хранят state в window._cianConfig['frontend-serp'].push(). +- URL pattern: https://ekb.cian.ru/cat.php?deal_type=sale&offer_type=flat&engine_version=2 + ekb.cian.ru — city-specific subdomain для ЕКБ (per Schema_Cian_SERP_Inventory sec 13). ВАЖНО: Циан блокирует httpx по TLS fingerprint → "Обнаружен подозрительный трафик" (403). Используем curl_cffi с impersonate='chrome120' — это libcurl-impersonate под капотом, который воспроизводит TLS ClientHello как настоящий Chrome. + +НЕ используем anchor jitter: offer.geo.coordinates.{lat,lng} — точные координаты +прямо из SERP state. Jitter запрещён per implementation plan. """ from __future__ import annotations -import json +import hashlib import logging -import random -import re +from datetime import UTC, datetime from typing import Any -from urllib.parse import urlencode, urljoin +from urllib.parse import urlencode from curl_cffi.requests import AsyncSession -from selectolax.parser import HTMLParser from app.services.scrapers.base import BaseScraper, ScrapedLot +from app.services.scrapers.cian_state_parser import extract_state logger = logging.getLogger(__name__) -# Регион 4743 = Свердловская область (Cian internal) +# Регион 4743 = Свердловская область (Cian internal region ID) CIAN_EKB_REGION_ID = 4743 +# SERP MFE name и state key (per Schema sec 1.2) +_MFE_SERP = "frontend-serp" +_STATE_KEY = "initialState" + class CianScraper(BaseScraper): - """Cian vtorichka parser. Использует curl_cffi для обхода TLS fingerprint detection.""" + """Cian SERP scraper. Использует curl_cffi для обхода TLS fingerprint. + + Извлекает 137 полей на offer из Redux initialState через cian_state_parser. + Координаты точные — НЕТ anchor jitter (offer.geo.coordinates). + """ name = "cian" - base_url = "https://ekaterinburg.cian.ru" - request_delay_sec = 6.0 + # ekb.cian.ru — city-specific subdomain для ЕКБ (per Schema sec 13, closed Q4) + base_url = "https://ekb.cian.ru" + request_delay_sec = 5.0 # консервативно: Cian менее агрессивен чем Avito, но 5s безопасно def __init__(self) -> None: super().__init__() self._cffi: AsyncSession | None = None - async def __aenter__(self) -> "CianScraper": + async def __aenter__(self) -> CianScraper: await super().__aenter__() - # curl_cffi session с Chrome 120 TLS fingerprint + # curl_cffi session с Chrome 120 TLS fingerprint — КРИТИЧЕСКИ ВАЖНО для Cian self._cffi = AsyncSession( impersonate="chrome120", - timeout=25, + timeout=30, headers={ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", @@ -65,42 +77,51 @@ class CianScraper(BaseScraper): await super().__aexit__(*args) async def fetch_around( - self, lat: float, lon: float, radius_m: int = 1000, - rooms: tuple[int, ...] | None = None, page: int = 1, + self, + lat: float, + lon: float, + radius_m: int = 1000, + rooms: tuple[int, ...] | None = None, + page: int = 1, ) -> list[ScrapedLot]: - """Найти объявления Циан вокруг (lat, lon) в radius_m метрах. + """Найти объявления Циан по ЕКБ (lat/lon для reference; Cian не поддерживает bbox). + + Cian SERP отдаёт весь ЕКБ — фильтрация по радиусу происходит в postgres + через ST_DWithin после сохранения (точные coords из state). rooms — список из 1,2,3,4 (Cian's room codes). Если None — все. - page — страница выдачи (Cian ~28 на страницу). + page — страница выдачи (Cian ~28 объявлений на страницу). """ - self._anchor_lat = lat - self._anchor_lon = lon - self._anchor_radius_m = radius_m - - url = self._build_url(lat, lon, radius_m, rooms, page) + url = self._build_url(rooms, page) try: assert self._cffi is not None response = await self._cffi.get(url) - except Exception: # noqa: BLE001 - logger.exception("cian curl_cffi fetch failed for %s", url) - return [] - if response.status_code != 200: - logger.warning("cian returned %d for %s", response.status_code, url) + except Exception: + logger.exception("cian curl_cffi fetch failed for url=%s", url) return [] - lots = self._parse_html(response.text) + if response.status_code != 200: + logger.warning("cian returned HTTP %d for url=%s", response.status_code, url) + return [] + + lots = self._parse_serp_html(response.text) logger.info( - "cian: %d lots around (%.4f, %.4f) rooms=%s page=%d", - len(lots), lat, lon, rooms, page, + "cian: %d lots fetched rooms=%s page=%d url=%s", + len(lots), rooms, page, url, ) await self.sleep_between_requests() return lots async def fetch_around_multi_room( - self, lat: float, lon: float, radius_m: int = 1000, pages: int = 8, + self, + lat: float, + lon: float, + radius_m: int = 1000, + pages: int = 8, ) -> list[ScrapedLot]: """Скрейп Циан по 1к/2к/3к/4+ × N страниц — расширяет выборку. - 4 комнаты × 3 страницы × ~28 = до ~330 лотов до дедупликации. + + 4 комнаты × N страниц × ~28 = до ~330+ лотов до дедупликации. """ seen: dict[str, ScrapedLot] = {} for rooms in ((1,), (2,), (3,), (4,)): @@ -113,28 +134,26 @@ class CianScraper(BaseScraper): ) continue if not lots: - break # пустая страница → дальше нет смысла + break # пустая страница → дальше смысла нет for lot in lots: key = lot.source_id or lot.source_url if key and key not in seen: seen[key] = lot logger.info( - "cian multi-room: %d unique lots around (%.4f, %.4f)", len(seen), lat, lon + "cian multi-room: %d unique lots (lat=%.4f lon=%.4f radius=%dm)", + len(seen), lat, lon, radius_m, ) return list(seen.values()) def _build_url( self, - lat: float, - lon: float, - radius_m: int, rooms: tuple[int, ...] | None = None, page: int = 1, ) -> str: - """URL для Cian каталога с регионом ЕКБ. + """URL для Cian каталога вторички ЕКБ. - Cian не имеет «радиус» в URL params, но мы фильтруем по lat/lon - в postgres через ST_DWithin позже. Сейчас тащим всё ЕКБ. + Используем ekb.cian.ru (city-specific subdomain). + Регион задаётся через region= param как fallback для reliability. """ params: list[tuple[str, Any]] = [ ("deal_type", "sale"), @@ -145,340 +164,320 @@ class CianScraper(BaseScraper): ] if rooms: for r in rooms: - # Cian rooms[]=1 для 1к, rooms[]=2 для 2к, и т.д. params.append((f"room{r}", "1")) if page > 1: params.append(("p", page)) return f"{self.base_url}/cat.php?{urlencode(params)}" - def _parse_html(self, html: str) -> list[ScrapedLot]: - """Cian вставляет state в `window.__NEXT_DATA__` или подобный JSON блок.""" - # Попытка 1: __NEXT_DATA__ script - items = _extract_next_data_items(html) - if items: - return [ - lot for lot in (self._item_to_lot(it) for it in items) if lot is not None - ] + def _parse_serp_html(self, html: str) -> list[ScrapedLot]: + """Извлечь offers из Cian Redux state. - # Попытка 2: regex-извлечение из inline JSON state - items = _extract_inline_offers(html) - if items: - return [ - lot for lot in (self._item_to_lot(it) for it in items) if lot is not None - ] - - # Попытка 3: DOM card scrape (fallback) - tree = HTMLParser(html) - cards = tree.css('[data-name="CardComponent"]') or tree.css( - 'article[data-name="GeneralCard"]' - ) - return [ - lot for lot in (self._card_to_lot(c) for c in cards) if lot is not None - ][:50] - - def _item_to_lot(self, item: dict[str, Any]) -> ScrapedLot | None: - """Парс Cian offer JSON → ScrapedLot.""" - try: - # Cian offer структура - offer_id = item.get("cianId") or item.get("id") - url = item.get("fullUrl") or item.get("url") - if not url and offer_id: - url = f"https://ekaterinburg.cian.ru/sale/flat/{offer_id}/" - if not url: - return None - - # Цена - bargain = item.get("bargainTerms") or {} - price = bargain.get("price") or item.get("price") - if not price: - return None - - # Параметры - rooms = item.get("roomsCount") - area = item.get("totalArea") or item.get("totalAreaSquare") - if area: - try: - area = float(area) - except (TypeError, ValueError): - area = None - floor = item.get("floorNumber") - building = item.get("building") or {} - total_floors = building.get("floorsCount") - - # Адрес - geo = item.get("geo") or {} - address_parts = geo.get("userInput") or "" - if isinstance(geo.get("address"), list): - address_parts = ", ".join( - a.get("fullName", "") or a.get("name", "") - for a in geo["address"][:3] - ) - - # Координаты - coords = geo.get("coordinates") or {} - lat = coords.get("lat") - lon = coords.get("lng") - - # Fallback anchor с jitter если нет точных - if lat is None or lon is None: - if hasattr(self, "_anchor_lat"): - jitter_deg = (self._anchor_radius_m / 1000) / 111 - lat = self._anchor_lat + (random.random() - 0.5) * jitter_deg - lon = self._anchor_lon + (random.random() - 0.5) * jitter_deg / 2 - - # Год постройки - year_built = building.get("buildYear") - - return ScrapedLot( - source="cian", - source_url=url, - source_id=str(offer_id) if offer_id else None, - address=address_parts or "Екатеринбург (Cian)", - lat=lat, - lon=lon, - rooms=rooms, - area_m2=area, - floor=floor, - total_floors=total_floors, - year_built=year_built, - price_rub=int(price), - photo_urls=[ - p.get("fullUrl") or p.get("thumbnailUrl", "") - for p in (item.get("photos") or [])[:5] - if isinstance(p, dict) - ], - raw_payload=item, + Использует cian_state_parser.extract_state() — общая утилита Stage 2. + Возвращает пустой список если state не найден. + """ + state = extract_state(html, mfe=_MFE_SERP, key=_STATE_KEY) + if state is None: + logger.warning( + "cian SERP state extraction failed (mfe=%s key=%s) — " + "возможно Cian изменил структуру или вернул captcha", + _MFE_SERP, _STATE_KEY, ) - except Exception: # noqa: BLE001 - logger.exception("cian item parse failed: %s", item.get("id")) - return None + return [] - def _card_to_lot(self, card: Any) -> ScrapedLot | None: - """DOM card-парсинг для Cian 2025+ верстки. + offers_data: list[dict[str, Any]] = ( + state.get("results", {}).get("offers", []) + ) + if not offers_data: + logger.warning( + "cian state found but results.offers пуст (totalOffers=%s)", + state.get("results", {}).get("totalOffers", "?"), + ) + return [] - Использует data-mark и data-name селекторы: - - [data-mark="OfferTitle"]: "2-комн. квартира, 47,5 м², 2/9 этаж" - - [data-mark="MainPrice"]: "6 000 000 ₽" - - [data-name="GeoLabel"]: address parts (multiple, объединяем) - - [data-mark="Description"]: long text (год постройки regex'ом) - - href контейнера → source_url + logger.info( + "cian SERP state ok: %d offers (totalOffers=%s)", + len(offers_data), + state.get("results", {}).get("totalOffers", "?"), + ) + + lots: list[ScrapedLot] = [] + for offer in offers_data: + lot = self._offer_to_lot(offer) + if lot is not None: + lots.append(lot) + return lots + + def _offer_to_lot(self, offer: dict[str, Any]) -> ScrapedLot | None: + """Парсинг одного Cian offer (137 fields) → ScrapedLot. + + Маппинг полей per Schema_Cian_SERP_Inventory sec 3, sec 8, sec 11. + Без anchor jitter: координаты из offer.geo.coordinates (точные). """ try: - link_el = card.css_first('a[href*="/sale/flat/"]') - if link_el is None: + # ── Identity ───────────────────────────────────────────────────── + offer_id = offer.get("cianId") or offer.get("id") + if not offer_id: + logger.debug("cian offer пропущен: нет cianId/id") return None - href = link_el.attributes.get("href", "") - url = urljoin(self.base_url, href) - # offer_id из URL pattern /sale/flat/NNN/ - offer_id_m = re.search(r"/sale/flat/(\d+)/", href) - offer_id = offer_id_m.group(1) if offer_id_m else None + source_id = str(offer_id) + url = offer.get("fullUrl") + if not url: + url = f"{self.base_url}/sale/flat/{offer_id}/" - # Цена - price_el = card.css_first('[data-mark="MainPrice"]') - price_text = price_el.text(strip=True) if price_el else "" - price = _extract_price_rub(price_text) + # ── Цена ───────────────────────────────────────────────────────── + bargain: dict[str, Any] = offer.get("bargainTerms") or {} + price = bargain.get("priceRur") or bargain.get("price") if not price: + logger.debug("cian offer %s пропущен: нет цены", offer_id) + return None + try: + price_rub = int(price) + except (TypeError, ValueError): + logger.debug("cian offer %s: не удалось распарсить цену %r", offer_id, price) + return None + if price_rub <= 0: return None - # Title → rooms / area / floor / total_floors - title_el = ( - card.css_first('[data-mark="OfferTitle"]') - or card.css_first('[data-name="TitleComponent"]') - or card.css_first('[data-mark="Title"]') - ) - title = title_el.text(strip=True) if title_el else "" - rooms = _extract_rooms_from_text(title) - area = _extract_area_from_text(title) - floor, total = _extract_floor_from_text(title) + # ── Параметры квартиры ─────────────────────────────────────────── + rooms: int | None = offer.get("roomsCount") + # Студия: flatType == 'studio' → 0 комнат + if offer.get("flatType") == "studio" and rooms is None: + rooms = 0 - # Address: собираем все GeoLabel, но показываем компактно - geo_parts = [ - el.text(strip=True) - for el in card.css('[data-name="GeoLabel"]') - if el.text(strip=True) + area_raw = offer.get("totalArea") + area_m2: float | None = None + if area_raw is not None: + try: + area_m2 = float(area_raw) + except (TypeError, ValueError): + pass + + living_area_raw = offer.get("livingArea") + living_area_m2: float | None = None + if living_area_raw is not None: + try: + living_area_m2 = float(living_area_raw) + except (TypeError, ValueError): + pass + + # kitchen_area_m2 сохраняем в raw_payload (поле не в ScrapedLot Stage 2) + kitchen_area_raw = offer.get("kitchenArea") + + floor: int | None = offer.get("floorNumber") + + # ── Здание ─────────────────────────────────────────────────────── + building: dict[str, Any] = offer.get("building") or {} + total_floors: int | None = building.get("floorsCount") + year_built: int | None = building.get("buildYear") + house_type: str | None = building.get("materialType") + + # Newbuilding deadline year — если нет buildYear + if year_built is None: + deadline = building.get("deadline") or {} + year_built = deadline.get("year") + + # ── Кадастр ───────────────────────────────────────────────────── + # cadastralNumber — кадастр КВАРТИРЫ + cadastral_number: str | None = offer.get("cadastralNumber") + # buildingCadastralNumber — кадастр ДОМА + building_cadastral_number: str | None = offer.get("buildingCadastralNumber") + + # ── Геолокация (точные координаты — NO jitter) ─────────────────── + geo: dict[str, Any] = offer.get("geo") or {} + coords: dict[str, Any] = geo.get("coordinates") or {} + lat: float | None = coords.get("lat") + lon: float | None = coords.get("lng") + # Если нет coords → сохраняем без lat/lon (geocode-missing обработает позже) + if lat is None or lon is None: + logger.debug( + "cian offer %s: нет точных координат в state (geo=%s)", + offer_id, coords, + ) + lat = lon = None + + # ── Адрес из geo.address[] ─────────────────────────────────────── + address_parts = geo.get("address") or [] + address = _format_address(address_parts) + + # ── Metro stations ─────────────────────────────────────────────── + undergrounds: list[dict[str, Any]] = geo.get("undergrounds") or [] + metro_stations = [ + { + "name": u.get("name"), + "time": u.get("time"), + "mode": u.get("transportType"), + "line_color": u.get("lineColor"), + "line_id": u.get("lineId"), + "is_default": u.get("isDefault", False), + } + for u in undergrounds + if u.get("name") ] - # Полный список сохраним в raw_payload, а display address укоротим. - # Убираем "Свердловская область", "Екатеринбург", "м. *" (метро), - # оставляем "р-н *", "мкр. *", улицу и дом. - full_geo = list(geo_parts) - display_parts: list[str] = [] - street_part: str | None = None - house_part: str | None = None - district_part: str | None = None - for p in geo_parts: - if p in {"Свердловская область", "Екатеринбург"}: - continue - if p.startswith("м. "): - continue # метро не нужно в адресе - if p.startswith("р-н ") and district_part is None: - district_part = p - elif p.startswith("мкр. "): - continue # микрорайон опускаем для компактности - elif "улица" in p or "проспект" in p or "переулок" in p or "бульвар" in p or "шоссе" in p: - street_part = p - elif re.match(r"^\d", p): - house_part = p - else: - display_parts.append(p) - # Финальный display: "улица X, дом Y · район Z" - final_parts: list[str] = [] - if street_part: - final_parts.append(street_part + (f", {house_part}" if house_part else "")) - if district_part: - final_parts.append(district_part) - address = " · ".join(final_parts) if final_parts else ( - ", ".join(p for p in geo_parts if p not in {"Свердловская область"}) or "Екатеринбург (Cian)" - ) - # Description → попытка извлечь год постройки - desc_el = card.css_first('[data-mark="Description"]') - desc = desc_el.text(strip=True) if desc_el else "" - year_built = _extract_year_built(desc) + # ── Newbuilding link ───────────────────────────────────────────── + newbuilding: dict[str, Any] = offer.get("newbuilding") or {} + nb_id = newbuilding.get("id") + # id == 0 — sentinel для вторички (per Schema sec 20.5) + house_source: str | None = None + house_ext_id: str | None = None + listing_segment: str | None = None - # PriceInfo: "126 316 ₽/м²" — можно посчитать area если её нет в title - if not area: - pinfo_el = card.css_first('[data-mark="PriceInfo"]') - pinfo = pinfo_el.text(strip=True) if pinfo_el else "" - m = re.search(r"([\d\s]+)\s*₽/м²", pinfo) - if m: - ppm2 = int(re.sub(r"\D", "", m.group(1)) or 0) - if ppm2 > 0: - area = round(price / ppm2, 2) + if nb_id and int(nb_id) > 0: + house_source = "cian_newbuilding" + house_ext_id = str(nb_id) + listing_segment = "novostroyki" + else: + house_source = "cian" + listing_segment = "vtorichka" - # Coords: НЕ ставим (Cian card HTML не отдаёт). Оставляем None — - # geocode-missing проставит реальные по address. Anchor+jitter тут - # был БАГ: Cian отдаёт ВСЕ объявления ЕКБ независимо от anchor, и - # jitter вокруг одного якоря давал фейковые координаты → ST_DWithin - # не находил аналоги для адресов вдали от центра. - lat = lon = None + # ── Дополнительные характеристики ──────────────────────────────── + balconies_count: int | None = offer.get("balconiesCount") + loggias_count: int | None = offer.get("loggiasCount") + bedrooms_count: int | None = offer.get("bedroomsCount") - # Photos из img.src в card (первые 5) - photo_urls: list[str] = [] - for img in card.css("img.x31de4314--_18b0f--container, [data-name=Cover] img")[:5]: - src = img.attributes.get("src", "") - if src and "cian" in src and src not in photo_urls: - photo_urls.append(src) + has_balcony: bool | None = None + if balconies_count is not None: + has_balcony = balconies_count > 0 + elif loggias_count is not None: + has_balcony = loggias_count > 0 + + # Мебель и repair_state + has_furniture: bool | None = offer.get("hasFurniture") or offer.get("isSoldFurnished") + # decoration — для новостроек (отделка); repair_state — для вторички + repair_state: str | None = offer.get("decoration") + + # ── Продавец ───────────────────────────────────────────────────── + phones: list[dict[str, Any]] = offer.get("phones") or [] + is_homeowner: bool | None = offer.get("isByHomeowner") + is_pro_seller: bool | None = offer.get("isPro") + + # ── Сделка ─────────────────────────────────────────────────────── + sale_type: str | None = bargain.get("saleType") + bargain_allowed: bool | None = bargain.get("bargainAllowed") + + # ── Description + minhash ───────────────────────────────────────── + description: str | None = offer.get("description") + # Cian предоставляет готовый minhash для dedup и cross-source matching + description_minhash: str | None = offer.get("descriptionMinhash") + # Если Cian не дал minhash — вычисляем простой SHA1 из description + if not description_minhash and description: + description_minhash = hashlib.sha1( + description.lower().encode("utf-8", errors="replace") + ).hexdigest()[:32] + + # ── Фото ───────────────────────────────────────────────────────── + photo_urls: list[str] = [ + p["fullUrl"] + for p in (offer.get("photos") or []) + if isinstance(p, dict) and p.get("fullUrl") + ] + + # ── Дата публикации ─────────────────────────────────────────────── + added_ts = offer.get("addedTimestamp") + listing_date = None + if added_ts: + try: + listing_date = datetime.fromtimestamp(int(added_ts), tz=UTC).date() + except (TypeError, ValueError, OSError): + pass + + # ── Raw payload (полный offer для enrichment в будущем) ─────────── + raw_payload: dict[str, Any] = { + "cian_id": offer_id, + "flat_type": offer.get("flatType"), + "is_apartments": offer.get("isApartments"), + "offer_type": offer.get("offerType"), + "category": offer.get("category"), + "has_furniture": has_furniture, + "kitchen_area_m2": kitchen_area_raw, # не в ScrapedLot — сохраняем здесь + "mortgage_allowed": bargain.get("mortgageAllowed"), + "sale_type": sale_type, + "newbuilding_name": newbuilding.get("name"), + "is_from_developer": ( + offer.get("fromDeveloper") or newbuilding.get("isFromDeveloper") + ), + "builders_ids": offer.get("buildersIds"), + "is_rosreestr_checked": offer.get("isRosreestrChecked"), + "is_layout_approved": offer.get("isLayoutApproved"), + "user_id": offer.get("userId"), + "published_user_id": offer.get("publishedUserId"), + "is_by_commercial_owner": offer.get("isByCommercialOwner"), + "is_cian_partner": offer.get("isCianPartner"), + "description_words_highlighted": offer.get("descriptionWordsHighlighted"), + "building_parking": building.get("parking"), + "building_total_area": building.get("totalArea"), + } return ScrapedLot( source="cian", source_url=url, - source_id=offer_id, + source_id=source_id, address=address, lat=lat, lon=lon, rooms=rooms, - area_m2=area, + area_m2=area_m2, floor=floor, - total_floors=total, + total_floors=total_floors, year_built=year_built, - price_rub=price, + house_type=house_type, + repair_state=repair_state, + has_balcony=has_balcony, + kadastr_num=cadastral_number, + house_source=house_source, + house_ext_id=house_ext_id, + listing_segment=listing_segment, + price_rub=price_rub, + price_per_m2=None, # compute_price_per_m2() вычислит + listing_date=listing_date, photo_urls=photo_urls, - raw_payload={ - "title": title, - "description": desc[:500], - "geo": full_geo, - }, + raw_payload=raw_payload, + # Cian-specific (Stage 2 fields) + living_area_m2=living_area_m2, + bedrooms_count=bedrooms_count, + balconies_count=balconies_count, + loggias_count=loggias_count, + description_minhash=description_minhash, + cadastral_number=cadastral_number, + building_cadastral_number=building_cadastral_number, + phones=phones, + is_homeowner=is_homeowner, + is_pro_seller=is_pro_seller, + bargain_allowed=bargain_allowed, + sale_type=sale_type, + metro_stations=metro_stations, ) - except Exception: # noqa: BLE001 - logger.exception("cian card parse failed") + + except Exception: + _oid = offer.get("cianId") or offer.get("id") + logger.exception("cian _offer_to_lot failed for offer_id=%s", _oid) return None -# ── HTML / JSON extraction helpers ────────────────────────────────────────── +# ── Address formatter ──────────────────────────────────────────────────────── -_RE_NEXT_DATA = re.compile( - r']+id="__NEXT_DATA__"[^>]*>(.+?)', re.DOTALL -) -_RE_INITIAL_STATE = re.compile( - r'window\._cianConfig\s*=\s*({.+?});', re.DOTALL -) +def _format_address(address_parts: list[dict[str, Any]]) -> str: + """Сформировать читаемый адрес из geo.address[] Cian. + Пропускаем location (страна/регион) и metro — берём раион/улицу/дом. + Пример: [location=Москва, raion=Пресненский, street=..., house=1С7, metro=Москва-Сити] + → 'Пресненский, улица ..., 1С7' + """ + if not address_parts: + return "Екатеринбург (Cian)" -def _extract_next_data_items(html: str) -> list[dict[str, Any]]: - """Cian Next.js приложение хранит state в `__NEXT_DATA__` JSON.""" - m = _RE_NEXT_DATA.search(html) - if not m: - return [] - try: - data = json.loads(m.group(1)) - except json.JSONDecodeError: - return [] - # Структура у Cian примерно такая: props.pageProps.initialState.serpData.offers - cursor: Any = data - for key in ["props", "pageProps", "initialState"]: - if not isinstance(cursor, dict): - return [] - cursor = cursor.get(key) - if not isinstance(cursor, dict): - return [] - serp = cursor.get("results") or cursor.get("serpData") or cursor.get("data") or {} - offers = serp.get("offersSerialized") or serp.get("offers") or [] - if isinstance(offers, list): - return offers - return [] - - -def _extract_inline_offers(html: str) -> list[dict[str, Any]]: - """Альтернативный поиск offers в любом большом JSON в HTML.""" - # Найти offers: [...] inline - for match in re.finditer(r'"offers":\[(.+?)\](?=,"|\})', html): - chunk = "[" + match.group(1) + "]" - try: - offers = json.loads(chunk) - if isinstance(offers, list) and offers and isinstance(offers[0], dict): - return offers - except json.JSONDecodeError: + skip_types = {"location", "metro"} + parts: list[str] = [] + for part in address_parts: + if not isinstance(part, dict): continue - return [] + ptype = part.get("type", "") + if ptype in skip_types: + continue + name = (part.get("fullName") or part.get("name") or "").strip() + if name: + parts.append(name) - -# ── Title parsers (Cian-specific format) ──────────────────────────────────── - - -_RE_ROOMS = re.compile(r"(\d)-комн", re.IGNORECASE) -_RE_STUDIO = re.compile(r"студи", re.IGNORECASE) -_RE_AREA = re.compile(r"(\d+[.,]?\d*)\s*м²", re.IGNORECASE) -_RE_FLOOR = re.compile(r"(\d+)\s*/\s*(\d+)\s*эт", re.IGNORECASE) -_RE_PRICE = re.compile(r"([\d\s]+?)\s*₽") - - -def _extract_rooms_from_text(text: str) -> int | None: - if _RE_STUDIO.search(text): - return 0 - m = _RE_ROOMS.search(text) - return int(m.group(1)) if m else None - - -def _extract_area_from_text(text: str) -> float | None: - m = _RE_AREA.search(text) - return float(m.group(1).replace(",", ".")) if m else None - - -def _extract_floor_from_text(text: str) -> tuple[int | None, int | None]: - m = _RE_FLOOR.search(text) - if m: - return int(m.group(1)), int(m.group(2)) - return None, None - - -def _extract_price_rub(text: str) -> int | None: - m = _RE_PRICE.search(text) - if m: - digits = re.sub(r"\D", "", m.group(1)) - return int(digits) if digits else None - return None - - -_RE_YEAR_BUILT = re.compile(r"(?:19[5-9]\d|20[0-2]\d)\s*(?:года?\s*постройки|г\.?\s*постройки)", re.IGNORECASE) - - -def _extract_year_built(text: str) -> int | None: - """Извлечь год постройки из description: '1988 года постройки' / '2005 г. постройки'.""" - m = _RE_YEAR_BUILT.search(text) - if m: - year_m = re.search(r"(19[5-9]\d|20[0-2]\d)", m.group(0)) - if year_m: - return int(year_m.group(1)) - return None + return ", ".join(parts) if parts else "Екатеринбург (Cian)" diff --git a/tradein-mvp/backend/tests/test_cian_serp_scraper.py b/tradein-mvp/backend/tests/test_cian_serp_scraper.py new file mode 100644 index 00000000..76a9567e --- /dev/null +++ b/tradein-mvp/backend/tests/test_cian_serp_scraper.py @@ -0,0 +1,458 @@ +"""Smoke тесты для cian.py SERP scraper (Stage 3 рефактор). + +Проверяют парсинг 137 полей из синтетического Redux state без сетевых запросов. +Fixture формирует минимальный JS push() conforming to _cianConfig pattern. +""" + +import json +import pytest + +from app.services.scrapers.cian import CianScraper, _format_address + + +# ── Fixture helpers ────────────────────────────────────────────────────────── + + +def _make_cian_serp_html(offers: list[dict]) -> str: + """Обернуть offers в валидный _cianConfig['frontend-serp'].push() HTML.""" + state = { + "results": { + "totalOffers": len(offers), + "offers": offers, + } + } + state_json = json.dumps(state, ensure_ascii=False) + return f""" + +""" + + +def _make_full_offer( + cian_id: int = 328739308, + price_rur: int = 6500000, + rooms: int = 2, + total_area: str = "52.0", + living_area: str = "35.0", + kitchen_area: str = "11.5", + floor: int = 7, + floors_count: int = 16, + year_built: int = 2010, + material_type: str = "panel", + lat: float = 56.8385, + lng: float = 60.5940, + cadastral_number: str = "66:41:0204016:1234", + building_cadastral_number: str = "66:41:0204016:100", + description: str = "Продаётся отличная квартира", + description_minhash: str = "abc123def456abc1", + newbuilding_id: int = 0, + has_phones: bool = True, + is_homeowner: bool = True, + is_pro: bool = False, + bargain_allowed: bool = True, + sale_type: str = "free", +) -> dict: + """Синтетический Cian offer с полным набором полей (137 полей subset).""" + offer: dict = { + "cianId": cian_id, + "id": cian_id, + "fullUrl": f"https://ekb.cian.ru/sale/flat/{cian_id}/", + "category": "flatSale", + "dealType": "sale", + "offerType": "flat", + "flatType": "rooms", + "isApartments": False, + "roomsCount": rooms, + "status": "published", + # Area + "totalArea": total_area, + "livingArea": living_area, + "kitchenArea": kitchen_area, + "balconiesCount": 1, + "loggiasCount": 0, + "bedroomsCount": None, + "hasFurniture": False, + "isSoldFurnished": False, + "decoration": None, + # Floor / building + "floorNumber": floor, + "building": { + "floorsCount": floors_count, + "materialType": material_type, + "buildYear": year_built, + "deadline": None, + "parking": None, + }, + # Cadastral + "cadastralNumber": cadastral_number, + "buildingCadastralNumber": building_cadastral_number, + # Price + "bargainTerms": { + "priceRur": price_rur, + "price": price_rur, + "currency": "rur", + "saleType": sale_type, + "mortgageAllowed": True, + "bargainAllowed": bargain_allowed, + "priceType": "all", + }, + "formattedShortPrice": f"{price_rur // 1000000} млн ₽", + # Geo + "geo": { + "coordinates": {"lat": lat, "lng": lng}, + "address": [ + {"type": "location", "name": "Свердловская область"}, + {"type": "city", "name": "Екатеринбург"}, + {"type": "street", "name": "улица Постовского"}, + {"type": "house", "name": "17А"}, + ], + "undergrounds": [ + { + "name": "Геологическая", + "time": 10, + "transportType": "walk", + "lineId": 10, + "lineColor": "FF0000", + "isDefault": True, + "id": 100, + } + ], + "districts": [], + }, + # Newbuilding + "newbuilding": {"id": newbuilding_id, "name": ""}, + "buildersIds": [], + # Seller + "phones": [{"number": "9012345678", "countryCode": "7"}] if has_phones else [], + "userId": 13332612, + "cianUserId": 13332612, + "publishedUserId": 13332612, + "isByHomeowner": is_homeowner, + "isByCommercialOwner": False, + "isCianPartner": False, + "isPro": is_pro, + "isCalltrackingEnabled": False, + # Description + "description": description, + "descriptionMinhash": description_minhash, + "isDuplicatedDescription": False, + # Time + "addedTimestamp": 1716451200, # 2024-05-23 00:00:00 UTC + "added": "вчера", + # Photos + "photos": [ + { + "id": 1001, + "fullUrl": f"https://images.cdn-cian.ru/images/{cian_id}-1.jpg", + "thumbnailUrl": f"https://images.cdn-cian.ru/images/{cian_id}-2.jpg", + } + ], + # Verification + "isRosreestrChecked": True, + "isLayoutApproved": None, + # Misc + "isNew": False, + "isTop3": False, + "isPremium": False, + } + return offer + + +# ── Tests ──────────────────────────────────────────────────────────────────── + + +def test_cian_serp_parses_full_offer(): + """Базовый тест: полный offer → ScrapedLot с ключевыми полями.""" + offer = _make_full_offer() + html = _make_cian_serp_html([offer]) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + + assert len(lots) == 1 + lot = lots[0] + + assert lot.source == "cian" + assert lot.source_id == "328739308" + assert lot.source_url == "https://ekb.cian.ru/sale/flat/328739308/" + + +def test_cian_serp_coords_no_jitter(): + """Координаты берутся точно из state — NO anchor jitter.""" + offer = _make_full_offer(lat=56.8385, lng=60.5940) + html = _make_cian_serp_html([offer]) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + + assert len(lots) == 1 + lot = lots[0] + assert lot.lat == pytest.approx(56.8385, abs=1e-9) + assert lot.lon == pytest.approx(60.5940, abs=1e-9) + + +def test_cian_serp_kitchen_area_in_raw_payload(): + """kitchen_area_m2 не в ScrapedLot, но сохраняется в raw_payload.""" + offer = _make_full_offer(kitchen_area="11.5") + html = _make_cian_serp_html([offer]) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + + lot = lots[0] + assert lot.raw_payload is not None + assert lot.raw_payload["kitchen_area_m2"] == "11.5" + + +def test_cian_serp_description_minhash(): + """description_minhash берётся из Cian state, если есть.""" + offer = _make_full_offer( + description_minhash="cian_minhash_abc123", + description="Тест описания", + ) + html = _make_cian_serp_html([offer]) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + + lot = lots[0] + assert lot.description_minhash == "cian_minhash_abc123" + + +def test_cian_serp_description_minhash_fallback(): + """Если Cian не дал minhash — вычисляем SHA1 из description.""" + offer = _make_full_offer(description_minhash="", description="Уникальное описание квартиры") + offer["descriptionMinhash"] = "" # явно пусто + html = _make_cian_serp_html([offer]) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + + lot = lots[0] + assert lot.description_minhash is not None + assert len(lot.description_minhash) == 32 # SHA1 hex [:32] + + +def test_cian_serp_phones(): + """phones — список dicts из state.""" + offer = _make_full_offer(has_phones=True) + html = _make_cian_serp_html([offer]) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + + lot = lots[0] + assert isinstance(lot.phones, list) + assert len(lot.phones) == 1 + assert lot.phones[0]["countryCode"] == "7" + assert lot.phones[0]["number"] == "9012345678" + + +def test_cian_serp_house_ext_id_newbuilding(): + """house_ext_id заполнен если offer.newbuilding.id > 0.""" + offer = _make_full_offer(newbuilding_id=54016) + html = _make_cian_serp_html([offer]) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + + lot = lots[0] + assert lot.house_source == "cian_newbuilding" + assert lot.house_ext_id == "54016" + assert lot.listing_segment == "novostroyki" + + +def test_cian_serp_vtorichka_no_house_ext_id(): + """Вторичка (newbuilding.id == 0) — house_ext_id None.""" + offer = _make_full_offer(newbuilding_id=0) + html = _make_cian_serp_html([offer]) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + + lot = lots[0] + assert lot.house_source == "cian" + assert lot.house_ext_id is None + assert lot.listing_segment == "vtorichka" + + +def test_cian_serp_metro_stations(): + """metro_stations — список dicts с name/time/mode/line_color.""" + offer = _make_full_offer() + html = _make_cian_serp_html([offer]) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + + lot = lots[0] + assert isinstance(lot.metro_stations, list) + assert len(lot.metro_stations) == 1 + station = lot.metro_stations[0] + assert station["name"] == "Геологическая" + assert station["time"] == 10 + assert station["mode"] == "walk" + assert station["line_color"] == "FF0000" + + +def test_cian_serp_cadastral_numbers(): + """Кадастровые номера квартиры и здания.""" + offer = _make_full_offer( + cadastral_number="66:41:0204016:1234", + building_cadastral_number="66:41:0204016:100", + ) + html = _make_cian_serp_html([offer]) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + + lot = lots[0] + assert lot.cadastral_number == "66:41:0204016:1234" + assert lot.building_cadastral_number == "66:41:0204016:100" + # kadastr_num (base field) тоже заполнен + assert lot.kadastr_num == "66:41:0204016:1234" + + +def test_cian_serp_is_homeowner_seller(): + """is_homeowner и is_pro_seller из offer.""" + offer = _make_full_offer(is_homeowner=True, is_pro=False) + html = _make_cian_serp_html([offer]) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + + lot = lots[0] + assert lot.is_homeowner is True + assert lot.is_pro_seller is False + + +def test_cian_serp_bargain_and_sale_type(): + """bargain_allowed и sale_type из bargainTerms.""" + offer = _make_full_offer(bargain_allowed=True, sale_type="alternative") + html = _make_cian_serp_html([offer]) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + + lot = lots[0] + assert lot.bargain_allowed is True + assert lot.sale_type == "alternative" + + +def test_cian_serp_building_fields(): + """Поля здания: year_built, house_type, total_floors.""" + offer = _make_full_offer(year_built=1988, material_type="brick", floors_count=9) + html = _make_cian_serp_html([offer]) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + + lot = lots[0] + assert lot.year_built == 1988 + assert lot.house_type == "brick" + assert lot.total_floors == 9 + + +def test_cian_serp_living_area(): + """living_area_m2 из offer.livingArea.""" + offer = _make_full_offer(living_area="35.0") + html = _make_cian_serp_html([offer]) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + + lot = lots[0] + assert lot.living_area_m2 == pytest.approx(35.0) + + +def test_cian_serp_state_missing_returns_empty(): + """Если state нет в HTML — возвращаем пустой список.""" + html = "

no cian state here

" + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + assert lots == [] + + +def test_cian_serp_skips_offer_without_price(): + """Offer без цены — пропускаем.""" + offer = _make_full_offer() + del offer["bargainTerms"]["priceRur"] + del offer["bargainTerms"]["price"] + html = _make_cian_serp_html([offer]) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + assert lots == [] + + +def test_cian_serp_multiple_offers(): + """Несколько offers → список ScrapedLot.""" + offers = [_make_full_offer(cian_id=100 + i) for i in range(5)] + html = _make_cian_serp_html(offers) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + assert len(lots) == 5 + ids = [lot.source_id for lot in lots] + assert sorted(ids) == ["100", "101", "102", "103", "104"] + + +def test_cian_serp_photo_urls(): + """photo_urls — полные URL из offer.photos[].fullUrl.""" + offer = _make_full_offer(cian_id=999) + html = _make_cian_serp_html([offer]) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + + lot = lots[0] + assert len(lot.photo_urls) == 1 + assert "cdn-cian.ru" in lot.photo_urls[0] + assert "999-1.jpg" in lot.photo_urls[0] + + +def test_cian_serp_no_coords_does_not_crash(): + """Если координаты отсутствуют — lot.lat/lon = None, не краш.""" + offer = _make_full_offer() + del offer["geo"]["coordinates"] + html = _make_cian_serp_html([offer]) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + + assert len(lots) == 1 + assert lots[0].lat is None + assert lots[0].lon is None + + +# ── Address formatter tests ────────────────────────────────────────────────── + + +def test_format_address_standard(): + """Стандартный geo.address[].""" + parts = [ + {"type": "location", "name": "Свердловская область"}, + {"type": "city", "name": "Екатеринбург"}, + {"type": "district", "name": "Чкаловский"}, + {"type": "street", "name": "улица Постовского"}, + {"type": "house", "name": "17А"}, + {"type": "metro", "name": "Геологическая"}, + ] + result = _format_address(parts) + # location и metro пропускаем + assert "Свердловская" not in result + assert "Геологическая" not in result + assert "Екатеринбург" in result + assert "17А" in result + + +def test_format_address_empty(): + """Пустой address[] → fallback string.""" + assert _format_address([]) == "Екатеринбург (Cian)"