tradein-mvp/backend/app/services/scrapers/cian.py

@@ -1,51 +1,63 @@
-"""Cian.ru scraper — парсинг вторички вокруг точки.
+"""Cian.ru scraper — state-based SERP parser (137 fields per offer).
 
-Стратегия:
-- Cian.ru Next.js приложение хранит state в `__NEXT_DATA__` JSON (script tag в HTML).
-- URL pattern: https://ekaterinburg.cian.ru/cat.php?deal_type=sale&offer_type=flat&engine_version=2
+Стратегия (Stage 3 рефактор):
+- Cian React micro-frontends хранят state в window._cianConfig['frontend-serp'].push().
+- URL pattern: https://ekb.cian.ru/cat.php?deal_type=sale&offer_type=flat&engine_version=2
+  ekb.cian.ru — city-specific subdomain для ЕКБ (per Schema_Cian_SERP_Inventory sec 13).
 
 ВАЖНО: Циан блокирует httpx по TLS fingerprint → "Обнаружен подозрительный трафик" (403).
 Используем curl_cffi с impersonate='chrome120' — это libcurl-impersonate под капотом,
 который воспроизводит TLS ClientHello как настоящий Chrome.
+
+НЕ используем anchor jitter: offer.geo.coordinates.{lat,lng} — точные координаты
+прямо из SERP state. Jitter запрещён per implementation plan.
 """
 
 from __future__ import annotations
 
-import json
+import hashlib
 import logging
-import random
-import re
+from datetime import UTC, datetime
 from typing import Any
-from urllib.parse import urlencode, urljoin
+from urllib.parse import urlencode
 
 from curl_cffi.requests import AsyncSession
-from selectolax.parser import HTMLParser
 
 from app.services.scrapers.base import BaseScraper, ScrapedLot
+from app.services.scrapers.cian_state_parser import extract_state
 
 logger = logging.getLogger(__name__)
 
-# Регион 4743 = Свердловская область (Cian internal)
+# Регион 4743 = Свердловская область (Cian internal region ID)
 CIAN_EKB_REGION_ID = 4743
 
+# SERP MFE name и state key (per Schema sec 1.2)
+_MFE_SERP = "frontend-serp"
+_STATE_KEY = "initialState"
+
 
 class CianScraper(BaseScraper):
-    """Cian vtorichka parser. Использует curl_cffi для обхода TLS fingerprint detection."""
+    """Cian SERP scraper. Использует curl_cffi для обхода TLS fingerprint.
+
+    Извлекает 137 полей на offer из Redux initialState через cian_state_parser.
+    Координаты точные — НЕТ anchor jitter (offer.geo.coordinates).
+    """
 
     name = "cian"
-    base_url = "https://ekaterinburg.cian.ru"
-    request_delay_sec = 6.0
+    # ekb.cian.ru — city-specific subdomain для ЕКБ (per Schema sec 13, closed Q4)
+    base_url = "https://ekb.cian.ru"
+    request_delay_sec = 5.0  # консервативно: Cian менее агрессивен чем Avito, но 5s безопасно
 
     def __init__(self) -> None:
         super().__init__()
         self._cffi: AsyncSession | None = None
 
-    async def __aenter__(self) -> "CianScraper":
+    async def __aenter__(self) -> CianScraper:
         await super().__aenter__()
-        # curl_cffi session с Chrome 120 TLS fingerprint
+        # curl_cffi session с Chrome 120 TLS fingerprint — КРИТИЧЕСКИ ВАЖНО для Cian
         self._cffi = AsyncSession(
             impersonate="chrome120",
-            timeout=25,
+            timeout=30,
             headers={
                 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                 "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
@@ -65,42 +77,51 @@ class CianScraper(BaseScraper):
         await super().__aexit__(*args)
 
     async def fetch_around(
-        self, lat: float, lon: float, radius_m: int = 1000,
-        rooms: tuple[int, ...] | None = None, page: int = 1,
+        self,
+        lat: float,
+        lon: float,
+        radius_m: int = 1000,
+        rooms: tuple[int, ...] | None = None,
+        page: int = 1,
     ) -> list[ScrapedLot]:
-        """Найти объявления Циан вокруг (lat, lon) в radius_m метрах.
+        """Найти объявления Циан по ЕКБ (lat/lon для reference; Cian не поддерживает bbox).
+
+        Cian SERP отдаёт весь ЕКБ — фильтрация по радиусу происходит в postgres
+        через ST_DWithin после сохранения (точные coords из state).
 
         rooms — список из 1,2,3,4 (Cian's room codes). Если None — все.
-        page — страница выдачи (Cian ~28 на страницу).
+        page — страница выдачи (Cian ~28 объявлений на страницу).
         """
-        self._anchor_lat = lat
-        self._anchor_lon = lon
-        self._anchor_radius_m = radius_m
-
-        url = self._build_url(lat, lon, radius_m, rooms, page)
+        url = self._build_url(rooms, page)
         try:
             assert self._cffi is not None
             response = await self._cffi.get(url)
-        except Exception:  # noqa: BLE001
-            logger.exception("cian curl_cffi fetch failed for %s", url)
-            return []
-        if response.status_code != 200:
-            logger.warning("cian returned %d for %s", response.status_code, url)
+        except Exception:
+            logger.exception("cian curl_cffi fetch failed for url=%s", url)
             return []
 
-        lots = self._parse_html(response.text)
+        if response.status_code != 200:
+            logger.warning("cian returned HTTP %d for url=%s", response.status_code, url)
+            return []
+
+        lots = self._parse_serp_html(response.text)
         logger.info(
-            "cian: %d lots around (%.4f, %.4f) rooms=%s page=%d",
-            len(lots), lat, lon, rooms, page,
+            "cian: %d lots fetched rooms=%s page=%d url=%s",
+            len(lots), rooms, page, url,
         )
         await self.sleep_between_requests()
         return lots
 
     async def fetch_around_multi_room(
-        self, lat: float, lon: float, radius_m: int = 1000, pages: int = 8,
+        self,
+        lat: float,
+        lon: float,
+        radius_m: int = 1000,
+        pages: int = 8,
     ) -> list[ScrapedLot]:
         """Скрейп Циан по 1к/2к/3к/4+ × N страниц — расширяет выборку.
-        4 комнаты × 3 страницы × ~28 = до ~330 лотов до дедупликации.
+
+        4 комнаты × N страниц × ~28 = до ~330+ лотов до дедупликации.
         """
         seen: dict[str, ScrapedLot] = {}
         for rooms in ((1,), (2,), (3,), (4,)):
@@ -113,28 +134,26 @@ class CianScraper(BaseScraper):
                     )
                     continue
                 if not lots:
-                    break  # пустая страница → дальше нет смысла
+                    break  # пустая страница → дальше смысла нет
                 for lot in lots:
                     key = lot.source_id or lot.source_url
                     if key and key not in seen:
                         seen[key] = lot
         logger.info(
-            "cian multi-room: %d unique lots around (%.4f, %.4f)", len(seen), lat, lon
+            "cian multi-room: %d unique lots (lat=%.4f lon=%.4f radius=%dm)",
+            len(seen), lat, lon, radius_m,
         )
         return list(seen.values())
 
     def _build_url(
         self,
-        lat: float,
-        lon: float,
-        radius_m: int,
         rooms: tuple[int, ...] | None = None,
         page: int = 1,
     ) -> str:
-        """URL для Cian каталога с регионом ЕКБ.
+        """URL для Cian каталога вторички ЕКБ.
 
-        Cian не имеет «радиус» в URL params, но мы фильтруем по lat/lon
-        в postgres через ST_DWithin позже. Сейчас тащим всё ЕКБ.
+        Используем ekb.cian.ru (city-specific subdomain).
+        Регион задаётся через region= param как fallback для reliability.
         """
         params: list[tuple[str, Any]] = [
             ("deal_type", "sale"),
@@ -145,340 +164,320 @@ class CianScraper(BaseScraper):
         ]
         if rooms:
             for r in rooms:
-                # Cian rooms[]=1 для 1к, rooms[]=2 для 2к, и т.д.
                 params.append((f"room{r}", "1"))
         if page > 1:
             params.append(("p", page))
         return f"{self.base_url}/cat.php?{urlencode(params)}"
 
-    def _parse_html(self, html: str) -> list[ScrapedLot]:
-        """Cian вставляет state в `window.__NEXT_DATA__` или подобный JSON блок."""
-        # Попытка 1: __NEXT_DATA__ script
-        items = _extract_next_data_items(html)
-        if items:
-            return [
-                lot for lot in (self._item_to_lot(it) for it in items) if lot is not None
-            ]
+    def _parse_serp_html(self, html: str) -> list[ScrapedLot]:
+        """Извлечь offers из Cian Redux state.
 
-        # Попытка 2: regex-извлечение из inline JSON state
-        items = _extract_inline_offers(html)
-        if items:
-            return [
-                lot for lot in (self._item_to_lot(it) for it in items) if lot is not None
-            ]
-
-        # Попытка 3: DOM card scrape (fallback)
-        tree = HTMLParser(html)
-        cards = tree.css('[data-name="CardComponent"]') or tree.css(
-            'article[data-name="GeneralCard"]'
-        )
-        return [
-            lot for lot in (self._card_to_lot(c) for c in cards) if lot is not None
-        ][:50]
-
-    def _item_to_lot(self, item: dict[str, Any]) -> ScrapedLot | None:
-        """Парс Cian offer JSON → ScrapedLot."""
-        try:
-            # Cian offer структура
-            offer_id = item.get("cianId") or item.get("id")
-            url = item.get("fullUrl") or item.get("url")
-            if not url and offer_id:
-                url = f"https://ekaterinburg.cian.ru/sale/flat/{offer_id}/"
-            if not url:
-                return None
-
-            # Цена
-            bargain = item.get("bargainTerms") or {}
-            price = bargain.get("price") or item.get("price")
-            if not price:
-                return None
-
-            # Параметры
-            rooms = item.get("roomsCount")
-            area = item.get("totalArea") or item.get("totalAreaSquare")
-            if area:
-                try:
-                    area = float(area)
-                except (TypeError, ValueError):
-                    area = None
-            floor = item.get("floorNumber")
-            building = item.get("building") or {}
-            total_floors = building.get("floorsCount")
-
-            # Адрес
-            geo = item.get("geo") or {}
-            address_parts = geo.get("userInput") or ""
-            if isinstance(geo.get("address"), list):
-                address_parts = ", ".join(
-                    a.get("fullName", "") or a.get("name", "")
-                    for a in geo["address"][:3]
-                )
-
-            # Координаты
-            coords = geo.get("coordinates") or {}
-            lat = coords.get("lat")
-            lon = coords.get("lng")
-
-            # Fallback anchor с jitter если нет точных
-            if lat is None or lon is None:
-                if hasattr(self, "_anchor_lat"):
-                    jitter_deg = (self._anchor_radius_m / 1000) / 111
-                    lat = self._anchor_lat + (random.random() - 0.5) * jitter_deg
-                    lon = self._anchor_lon + (random.random() - 0.5) * jitter_deg / 2
-
-            # Год постройки
-            year_built = building.get("buildYear")
-
-            return ScrapedLot(
-                source="cian",
-                source_url=url,
-                source_id=str(offer_id) if offer_id else None,
-                address=address_parts or "Екатеринбург (Cian)",
-                lat=lat,
-                lon=lon,
-                rooms=rooms,
-                area_m2=area,
-                floor=floor,
-                total_floors=total_floors,
-                year_built=year_built,
-                price_rub=int(price),
-                photo_urls=[
-                    p.get("fullUrl") or p.get("thumbnailUrl", "")
-                    for p in (item.get("photos") or [])[:5]
-                    if isinstance(p, dict)
-                ],
-                raw_payload=item,
+        Использует cian_state_parser.extract_state() — общая утилита Stage 2.
+        Возвращает пустой список если state не найден.
+        """
+        state = extract_state(html, mfe=_MFE_SERP, key=_STATE_KEY)
+        if state is None:
+            logger.warning(
+                "cian SERP state extraction failed (mfe=%s key=%s) — "
+                "возможно Cian изменил структуру или вернул captcha",
+                _MFE_SERP, _STATE_KEY,
             )
-        except Exception:  # noqa: BLE001
-            logger.exception("cian item parse failed: %s", item.get("id"))
-            return None
+            return []
 
-    def _card_to_lot(self, card: Any) -> ScrapedLot | None:
-        """DOM card-парсинг для Cian 2025+ верстки.
+        offers_data: list[dict[str, Any]] = (
+            state.get("results", {}).get("offers", [])
+        )
+        if not offers_data:
+            logger.warning(
+                "cian state found but results.offers пуст (totalOffers=%s)",
+                state.get("results", {}).get("totalOffers", "?"),
+            )
+            return []
 
-        Использует data-mark и data-name селекторы:
-        - [data-mark="OfferTitle"]: "2-комн. квартира, 47,5 м², 2/9 этаж"
-        - [data-mark="MainPrice"]: "6 000 000 ₽"
-        - [data-name="GeoLabel"]: address parts (multiple, объединяем)
-        - [data-mark="Description"]: long text (год постройки regex'ом)
-        - href контейнера → source_url
+        logger.info(
+            "cian SERP state ok: %d offers (totalOffers=%s)",
+            len(offers_data),
+            state.get("results", {}).get("totalOffers", "?"),
+        )
+
+        lots: list[ScrapedLot] = []
+        for offer in offers_data:
+            lot = self._offer_to_lot(offer)
+            if lot is not None:
+                lots.append(lot)
+        return lots
+
+    def _offer_to_lot(self, offer: dict[str, Any]) -> ScrapedLot | None:
+        """Парсинг одного Cian offer (137 fields) → ScrapedLot.
+
+        Маппинг полей per Schema_Cian_SERP_Inventory sec 3, sec 8, sec 11.
+        Без anchor jitter: координаты из offer.geo.coordinates (точные).
         """
         try:
-            link_el = card.css_first('a[href*="/sale/flat/"]')
-            if link_el is None:
+            # ── Identity ─────────────────────────────────────────────────────
+            offer_id = offer.get("cianId") or offer.get("id")
+            if not offer_id:
+                logger.debug("cian offer пропущен: нет cianId/id")
                 return None
-            href = link_el.attributes.get("href", "")
-            url = urljoin(self.base_url, href)
 
-            # offer_id из URL pattern /sale/flat/NNN/
-            offer_id_m = re.search(r"/sale/flat/(\d+)/", href)
-            offer_id = offer_id_m.group(1) if offer_id_m else None
+            source_id = str(offer_id)
+            url = offer.get("fullUrl")
+            if not url:
+                url = f"{self.base_url}/sale/flat/{offer_id}/"
 
-            # Цена
-            price_el = card.css_first('[data-mark="MainPrice"]')
-            price_text = price_el.text(strip=True) if price_el else ""
-            price = _extract_price_rub(price_text)
+            # ── Цена ─────────────────────────────────────────────────────────
+            bargain: dict[str, Any] = offer.get("bargainTerms") or {}
+            price = bargain.get("priceRur") or bargain.get("price")
             if not price:
+                logger.debug("cian offer %s пропущен: нет цены", offer_id)
+                return None
+            try:
+                price_rub = int(price)
+            except (TypeError, ValueError):
+                logger.debug("cian offer %s: не удалось распарсить цену %r", offer_id, price)
+                return None
+            if price_rub <= 0:
                 return None
 
-            # Title → rooms / area / floor / total_floors
-            title_el = (
-                card.css_first('[data-mark="OfferTitle"]')
-                or card.css_first('[data-name="TitleComponent"]')
-                or card.css_first('[data-mark="Title"]')
-            )
-            title = title_el.text(strip=True) if title_el else ""
-            rooms = _extract_rooms_from_text(title)
-            area = _extract_area_from_text(title)
-            floor, total = _extract_floor_from_text(title)
+            # ── Параметры квартиры ───────────────────────────────────────────
+            rooms: int | None = offer.get("roomsCount")
+            # Студия: flatType == 'studio' → 0 комнат
+            if offer.get("flatType") == "studio" and rooms is None:
+                rooms = 0
 
-            # Address: собираем все GeoLabel, но показываем компактно
-            geo_parts = [
-                el.text(strip=True)
-                for el in card.css('[data-name="GeoLabel"]')
-                if el.text(strip=True)
+            area_raw = offer.get("totalArea")
+            area_m2: float | None = None
+            if area_raw is not None:
+                try:
+                    area_m2 = float(area_raw)
+                except (TypeError, ValueError):
+                    pass
+
+            living_area_raw = offer.get("livingArea")
+            living_area_m2: float | None = None
+            if living_area_raw is not None:
+                try:
+                    living_area_m2 = float(living_area_raw)
+                except (TypeError, ValueError):
+                    pass
+
+            # kitchen_area_m2 сохраняем в raw_payload (поле не в ScrapedLot Stage 2)
+            kitchen_area_raw = offer.get("kitchenArea")
+
+            floor: int | None = offer.get("floorNumber")
+
+            # ── Здание ───────────────────────────────────────────────────────
+            building: dict[str, Any] = offer.get("building") or {}
+            total_floors: int | None = building.get("floorsCount")
+            year_built: int | None = building.get("buildYear")
+            house_type: str | None = building.get("materialType")
+
+            # Newbuilding deadline year — если нет buildYear
+            if year_built is None:
+                deadline = building.get("deadline") or {}
+                year_built = deadline.get("year")
+
+            # ── Кадастр ─────────────────────────────────────────────────────
+            # cadastralNumber — кадастр КВАРТИРЫ
+            cadastral_number: str | None = offer.get("cadastralNumber")
+            # buildingCadastralNumber — кадастр ДОМА
+            building_cadastral_number: str | None = offer.get("buildingCadastralNumber")
+
+            # ── Геолокация (точные координаты — NO jitter) ───────────────────
+            geo: dict[str, Any] = offer.get("geo") or {}
+            coords: dict[str, Any] = geo.get("coordinates") or {}
+            lat: float | None = coords.get("lat")
+            lon: float | None = coords.get("lng")
+            # Если нет coords → сохраняем без lat/lon (geocode-missing обработает позже)
+            if lat is None or lon is None:
+                logger.debug(
+                    "cian offer %s: нет точных координат в state (geo=%s)",
+                    offer_id, coords,
+                )
+                lat = lon = None
+
+            # ── Адрес из geo.address[] ───────────────────────────────────────
+            address_parts = geo.get("address") or []
+            address = _format_address(address_parts)
+
+            # ── Metro stations ───────────────────────────────────────────────
+            undergrounds: list[dict[str, Any]] = geo.get("undergrounds") or []
+            metro_stations = [
+                {
+                    "name": u.get("name"),
+                    "time": u.get("time"),
+                    "mode": u.get("transportType"),
+                    "line_color": u.get("lineColor"),
+                    "line_id": u.get("lineId"),
+                    "is_default": u.get("isDefault", False),
+                }
+                for u in undergrounds
+                if u.get("name")
             ]
-            # Полный список сохраним в raw_payload, а display address укоротим.
-            # Убираем "Свердловская область", "Екатеринбург", "м. *" (метро),
-            # оставляем "р-н *", "мкр. *", улицу и дом.
-            full_geo = list(geo_parts)
-            display_parts: list[str] = []
-            street_part: str | None = None
-            house_part: str | None = None
-            district_part: str | None = None
-            for p in geo_parts:
-                if p in {"Свердловская область", "Екатеринбург"}:
-                    continue
-                if p.startswith("м. "):
-                    continue  # метро не нужно в адресе
-                if p.startswith("р-н ") and district_part is None:
-                    district_part = p
-                elif p.startswith("мкр. "):
-                    continue  # микрорайон опускаем для компактности
-                elif "улица" in p or "проспект" in p or "переулок" in p or "бульвар" in p or "шоссе" in p:
-                    street_part = p
-                elif re.match(r"^\d", p):
-                    house_part = p
-                else:
-                    display_parts.append(p)
-            # Финальный display: "улица X, дом Y · район Z"
-            final_parts: list[str] = []
-            if street_part:
-                final_parts.append(street_part + (f", {house_part}" if house_part else ""))
-            if district_part:
-                final_parts.append(district_part)
-            address = " · ".join(final_parts) if final_parts else (
-                ", ".join(p for p in geo_parts if p not in {"Свердловская область"}) or "Екатеринбург (Cian)"
-            )
 
-            # Description → попытка извлечь год постройки
-            desc_el = card.css_first('[data-mark="Description"]')
-            desc = desc_el.text(strip=True) if desc_el else ""
-            year_built = _extract_year_built(desc)
+            # ── Newbuilding link ─────────────────────────────────────────────
+            newbuilding: dict[str, Any] = offer.get("newbuilding") or {}
+            nb_id = newbuilding.get("id")
+            # id == 0 — sentinel для вторички (per Schema sec 20.5)
+            house_source: str | None = None
+            house_ext_id: str | None = None
+            listing_segment: str | None = None
 
-            # PriceInfo: "126 316 ₽/м²" — можно посчитать area если её нет в title
-            if not area:
-                pinfo_el = card.css_first('[data-mark="PriceInfo"]')
-                pinfo = pinfo_el.text(strip=True) if pinfo_el else ""
-                m = re.search(r"([\d\s]+)\s*₽/м²", pinfo)
-                if m:
-                    ppm2 = int(re.sub(r"\D", "", m.group(1)) or 0)
-                    if ppm2 > 0:
-                        area = round(price / ppm2, 2)
+            if nb_id and int(nb_id) > 0:
+                house_source = "cian_newbuilding"
+                house_ext_id = str(nb_id)
+                listing_segment = "novostroyki"
+            else:
+                house_source = "cian"
+                listing_segment = "vtorichka"
 
-            # Coords: НЕ ставим (Cian card HTML не отдаёт). Оставляем None —
-            # geocode-missing проставит реальные по address. Anchor+jitter тут
-            # был БАГ: Cian отдаёт ВСЕ объявления ЕКБ независимо от anchor, и
-            # jitter вокруг одного якоря давал фейковые координаты → ST_DWithin
-            # не находил аналоги для адресов вдали от центра.
-            lat = lon = None
+            # ── Дополнительные характеристики ────────────────────────────────
+            balconies_count: int | None = offer.get("balconiesCount")
+            loggias_count: int | None = offer.get("loggiasCount")
+            bedrooms_count: int | None = offer.get("bedroomsCount")
 
-            # Photos из img.src в card (первые 5)
-            photo_urls: list[str] = []
-            for img in card.css("img.x31de4314--_18b0f--container, [data-name=Cover] img")[:5]:
-                src = img.attributes.get("src", "")
-                if src and "cian" in src and src not in photo_urls:
-                    photo_urls.append(src)
+            has_balcony: bool | None = None
+            if balconies_count is not None:
+                has_balcony = balconies_count > 0
+            elif loggias_count is not None:
+                has_balcony = loggias_count > 0
+
+            # Мебель и repair_state
+            has_furniture: bool | None = offer.get("hasFurniture") or offer.get("isSoldFurnished")
+            # decoration — для новостроек (отделка); repair_state — для вторички
+            repair_state: str | None = offer.get("decoration")
+
+            # ── Продавец ─────────────────────────────────────────────────────
+            phones: list[dict[str, Any]] = offer.get("phones") or []
+            is_homeowner: bool | None = offer.get("isByHomeowner")
+            is_pro_seller: bool | None = offer.get("isPro")
+
+            # ── Сделка ───────────────────────────────────────────────────────
+            sale_type: str | None = bargain.get("saleType")
+            bargain_allowed: bool | None = bargain.get("bargainAllowed")
+
+            # ── Description + minhash ─────────────────────────────────────────
+            description: str | None = offer.get("description")
+            # Cian предоставляет готовый minhash для dedup и cross-source matching
+            description_minhash: str | None = offer.get("descriptionMinhash")
+            # Если Cian не дал minhash — вычисляем простой SHA1 из description
+            if not description_minhash and description:
+                description_minhash = hashlib.sha1(
+                    description.lower().encode("utf-8", errors="replace")
+                ).hexdigest()[:32]
+
+            # ── Фото ─────────────────────────────────────────────────────────
+            photo_urls: list[str] = [
+                p["fullUrl"]
+                for p in (offer.get("photos") or [])
+                if isinstance(p, dict) and p.get("fullUrl")
+            ]
+
+            # ── Дата публикации ───────────────────────────────────────────────
+            added_ts = offer.get("addedTimestamp")
+            listing_date = None
+            if added_ts:
+                try:
+                    listing_date = datetime.fromtimestamp(int(added_ts), tz=UTC).date()
+                except (TypeError, ValueError, OSError):
+                    pass
+
+            # ── Raw payload (полный offer для enrichment в будущем) ───────────
+            raw_payload: dict[str, Any] = {
+                "cian_id": offer_id,
+                "flat_type": offer.get("flatType"),
+                "is_apartments": offer.get("isApartments"),
+                "offer_type": offer.get("offerType"),
+                "category": offer.get("category"),
+                "has_furniture": has_furniture,
+                "kitchen_area_m2": kitchen_area_raw,  # не в ScrapedLot — сохраняем здесь
+                "mortgage_allowed": bargain.get("mortgageAllowed"),
+                "sale_type": sale_type,
+                "newbuilding_name": newbuilding.get("name"),
+                "is_from_developer": (
+                    offer.get("fromDeveloper") or newbuilding.get("isFromDeveloper")
+                ),
+                "builders_ids": offer.get("buildersIds"),
+                "is_rosreestr_checked": offer.get("isRosreestrChecked"),
+                "is_layout_approved": offer.get("isLayoutApproved"),
+                "user_id": offer.get("userId"),
+                "published_user_id": offer.get("publishedUserId"),
+                "is_by_commercial_owner": offer.get("isByCommercialOwner"),
+                "is_cian_partner": offer.get("isCianPartner"),
+                "description_words_highlighted": offer.get("descriptionWordsHighlighted"),
+                "building_parking": building.get("parking"),
+                "building_total_area": building.get("totalArea"),
+            }
 
             return ScrapedLot(
                 source="cian",
                 source_url=url,
-                source_id=offer_id,
+                source_id=source_id,
                 address=address,
                 lat=lat,
                 lon=lon,
                 rooms=rooms,
-                area_m2=area,
+                area_m2=area_m2,
                 floor=floor,
-                total_floors=total,
+                total_floors=total_floors,
                 year_built=year_built,
-                price_rub=price,
+                house_type=house_type,
+                repair_state=repair_state,
+                has_balcony=has_balcony,
+                kadastr_num=cadastral_number,
+                house_source=house_source,
+                house_ext_id=house_ext_id,
+                listing_segment=listing_segment,
+                price_rub=price_rub,
+                price_per_m2=None,  # compute_price_per_m2() вычислит
+                listing_date=listing_date,
                 photo_urls=photo_urls,
-                raw_payload={
-                    "title": title,
-                    "description": desc[:500],
-                    "geo": full_geo,
-                },
+                raw_payload=raw_payload,
+                # Cian-specific (Stage 2 fields)
+                living_area_m2=living_area_m2,
+                bedrooms_count=bedrooms_count,
+                balconies_count=balconies_count,
+                loggias_count=loggias_count,
+                description_minhash=description_minhash,
+                cadastral_number=cadastral_number,
+                building_cadastral_number=building_cadastral_number,
+                phones=phones,
+                is_homeowner=is_homeowner,
+                is_pro_seller=is_pro_seller,
+                bargain_allowed=bargain_allowed,
+                sale_type=sale_type,
+                metro_stations=metro_stations,
             )
-        except Exception:  # noqa: BLE001
-            logger.exception("cian card parse failed")
+
+        except Exception:
+            _oid = offer.get("cianId") or offer.get("id")
+            logger.exception("cian _offer_to_lot failed for offer_id=%s", _oid)
             return None
 
 
-# ── HTML / JSON extraction helpers ──────────────────────────────────────────
+# ── Address formatter ────────────────────────────────────────────────────────
 
 
-_RE_NEXT_DATA = re.compile(
-    r'<script[^>]+id="__NEXT_DATA__"[^>]*>(.+?)</script>', re.DOTALL
-)
-_RE_INITIAL_STATE = re.compile(
-    r'window\._cianConfig\s*=\s*({.+?});</script>', re.DOTALL
-)
+def _format_address(address_parts: list[dict[str, Any]]) -> str:
+    """Сформировать читаемый адрес из geo.address[] Cian.
 
+    Пропускаем location (страна/регион) и metro — берём раион/улицу/дом.
+    Пример: [location=Москва, raion=Пресненский, street=..., house=1С7, metro=Москва-Сити]
+    → 'Пресненский, улица ..., 1С7'
+    """
+    if not address_parts:
+        return "Екатеринбург (Cian)"
 
-def _extract_next_data_items(html: str) -> list[dict[str, Any]]:
-    """Cian Next.js приложение хранит state в `__NEXT_DATA__` JSON."""
-    m = _RE_NEXT_DATA.search(html)
-    if not m:
-        return []
-    try:
-        data = json.loads(m.group(1))
-    except json.JSONDecodeError:
-        return []
-    # Структура у Cian примерно такая: props.pageProps.initialState.serpData.offers
-    cursor: Any = data
-    for key in ["props", "pageProps", "initialState"]:
-        if not isinstance(cursor, dict):
-            return []
-        cursor = cursor.get(key)
-    if not isinstance(cursor, dict):
-        return []
-    serp = cursor.get("results") or cursor.get("serpData") or cursor.get("data") or {}
-    offers = serp.get("offersSerialized") or serp.get("offers") or []
-    if isinstance(offers, list):
-        return offers
-    return []
-
-
-def _extract_inline_offers(html: str) -> list[dict[str, Any]]:
-    """Альтернативный поиск offers в любом большом JSON в HTML."""
-    # Найти offers: [...] inline
-    for match in re.finditer(r'"offers":\[(.+?)\](?=,"|\})', html):
-        chunk = "[" + match.group(1) + "]"
-        try:
-            offers = json.loads(chunk)
-            if isinstance(offers, list) and offers and isinstance(offers[0], dict):
-                return offers
-        except json.JSONDecodeError:
+    skip_types = {"location", "metro"}
+    parts: list[str] = []
+    for part in address_parts:
+        if not isinstance(part, dict):
             continue
-    return []
+        ptype = part.get("type", "")
+        if ptype in skip_types:
+            continue
+        name = (part.get("fullName") or part.get("name") or "").strip()
+        if name:
+            parts.append(name)
 
-
-# ── Title parsers (Cian-specific format) ────────────────────────────────────
-
-
-_RE_ROOMS = re.compile(r"(\d)-комн", re.IGNORECASE)
-_RE_STUDIO = re.compile(r"студи", re.IGNORECASE)
-_RE_AREA = re.compile(r"(\d+[.,]?\d*)\s*м²", re.IGNORECASE)
-_RE_FLOOR = re.compile(r"(\d+)\s*/\s*(\d+)\s*эт", re.IGNORECASE)
-_RE_PRICE = re.compile(r"([\d\s]+?)\s*₽")
-
-
-def _extract_rooms_from_text(text: str) -> int | None:
-    if _RE_STUDIO.search(text):
-        return 0
-    m = _RE_ROOMS.search(text)
-    return int(m.group(1)) if m else None
-
-
-def _extract_area_from_text(text: str) -> float | None:
-    m = _RE_AREA.search(text)
-    return float(m.group(1).replace(",", ".")) if m else None
-
-
-def _extract_floor_from_text(text: str) -> tuple[int | None, int | None]:
-    m = _RE_FLOOR.search(text)
-    if m:
-        return int(m.group(1)), int(m.group(2))
-    return None, None
-
-
-def _extract_price_rub(text: str) -> int | None:
-    m = _RE_PRICE.search(text)
-    if m:
-        digits = re.sub(r"\D", "", m.group(1))
-        return int(digits) if digits else None
-    return None
-
-
-_RE_YEAR_BUILT = re.compile(r"(?:19[5-9]\d|20[0-2]\d)\s*(?:года?\s*постройки|г\.?\s*постройки)", re.IGNORECASE)
-
-
-def _extract_year_built(text: str) -> int | None:
-    """Извлечь год постройки из description: '1988 года постройки' / '2005 г. постройки'."""
-    m = _RE_YEAR_BUILT.search(text)
-    if m:
-        year_m = re.search(r"(19[5-9]\d|20[0-2]\d)", m.group(0))
-        if year_m:
-            return int(year_m.group(1))
-    return None
+    return ", ".join(parts) if parts else "Екатеринбург (Cian)"

tradein-mvp/backend/tests/test_cian_serp_scraper.py

@@ -0,0 +1,458 @@
+"""Smoke тесты для cian.py SERP scraper (Stage 3 рефактор).
+
+Проверяют парсинг 137 полей из синтетического Redux state без сетевых запросов.
+Fixture формирует минимальный JS push() conforming to _cianConfig pattern.
+"""
+
+import json
+import pytest
+
+from app.services.scrapers.cian import CianScraper, _format_address
+
+
+# ── Fixture helpers ──────────────────────────────────────────────────────────
+
+
+def _make_cian_serp_html(offers: list[dict]) -> str:
+    """Обернуть offers в валидный _cianConfig['frontend-serp'].push() HTML."""
+    state = {
+        "results": {
+            "totalOffers": len(offers),
+            "offers": offers,
+        }
+    }
+    state_json = json.dumps(state, ensure_ascii=False)
+    return f"""<html><body>
+<script>
+window._cianConfig['frontend-serp'] = window._cianConfig['frontend-serp'] || [];
+window._cianConfig['frontend-serp'].push({{
+    key: 'initialState',
+    value: {state_json},
+    priority: 10,
+    filter: null
+}});
+</script>
+</body></html>"""
+
+
+def _make_full_offer(
+    cian_id: int = 328739308,
+    price_rur: int = 6500000,
+    rooms: int = 2,
+    total_area: str = "52.0",
+    living_area: str = "35.0",
+    kitchen_area: str = "11.5",
+    floor: int = 7,
+    floors_count: int = 16,
+    year_built: int = 2010,
+    material_type: str = "panel",
+    lat: float = 56.8385,
+    lng: float = 60.5940,
+    cadastral_number: str = "66:41:0204016:1234",
+    building_cadastral_number: str = "66:41:0204016:100",
+    description: str = "Продаётся отличная квартира",
+    description_minhash: str = "abc123def456abc1",
+    newbuilding_id: int = 0,
+    has_phones: bool = True,
+    is_homeowner: bool = True,
+    is_pro: bool = False,
+    bargain_allowed: bool = True,
+    sale_type: str = "free",
+) -> dict:
+    """Синтетический Cian offer с полным набором полей (137 полей subset)."""
+    offer: dict = {
+        "cianId": cian_id,
+        "id": cian_id,
+        "fullUrl": f"https://ekb.cian.ru/sale/flat/{cian_id}/",
+        "category": "flatSale",
+        "dealType": "sale",
+        "offerType": "flat",
+        "flatType": "rooms",
+        "isApartments": False,
+        "roomsCount": rooms,
+        "status": "published",
+        # Area
+        "totalArea": total_area,
+        "livingArea": living_area,
+        "kitchenArea": kitchen_area,
+        "balconiesCount": 1,
+        "loggiasCount": 0,
+        "bedroomsCount": None,
+        "hasFurniture": False,
+        "isSoldFurnished": False,
+        "decoration": None,
+        # Floor / building
+        "floorNumber": floor,
+        "building": {
+            "floorsCount": floors_count,
+            "materialType": material_type,
+            "buildYear": year_built,
+            "deadline": None,
+            "parking": None,
+        },
+        # Cadastral
+        "cadastralNumber": cadastral_number,
+        "buildingCadastralNumber": building_cadastral_number,
+        # Price
+        "bargainTerms": {
+            "priceRur": price_rur,
+            "price": price_rur,
+            "currency": "rur",
+            "saleType": sale_type,
+            "mortgageAllowed": True,
+            "bargainAllowed": bargain_allowed,
+            "priceType": "all",
+        },
+        "formattedShortPrice": f"{price_rur // 1000000} млн ₽",
+        # Geo
+        "geo": {
+            "coordinates": {"lat": lat, "lng": lng},
+            "address": [
+                {"type": "location", "name": "Свердловская область"},
+                {"type": "city", "name": "Екатеринбург"},
+                {"type": "street", "name": "улица Постовского"},
+                {"type": "house", "name": "17А"},
+            ],
+            "undergrounds": [
+                {
+                    "name": "Геологическая",
+                    "time": 10,
+                    "transportType": "walk",
+                    "lineId": 10,
+                    "lineColor": "FF0000",
+                    "isDefault": True,
+                    "id": 100,
+                }
+            ],
+            "districts": [],
+        },
+        # Newbuilding
+        "newbuilding": {"id": newbuilding_id, "name": ""},
+        "buildersIds": [],
+        # Seller
+        "phones": [{"number": "9012345678", "countryCode": "7"}] if has_phones else [],
+        "userId": 13332612,
+        "cianUserId": 13332612,
+        "publishedUserId": 13332612,
+        "isByHomeowner": is_homeowner,
+        "isByCommercialOwner": False,
+        "isCianPartner": False,
+        "isPro": is_pro,
+        "isCalltrackingEnabled": False,
+        # Description
+        "description": description,
+        "descriptionMinhash": description_minhash,
+        "isDuplicatedDescription": False,
+        # Time
+        "addedTimestamp": 1716451200,  # 2024-05-23 00:00:00 UTC
+        "added": "вчера",
+        # Photos
+        "photos": [
+            {
+                "id": 1001,
+                "fullUrl": f"https://images.cdn-cian.ru/images/{cian_id}-1.jpg",
+                "thumbnailUrl": f"https://images.cdn-cian.ru/images/{cian_id}-2.jpg",
+            }
+        ],
+        # Verification
+        "isRosreestrChecked": True,
+        "isLayoutApproved": None,
+        # Misc
+        "isNew": False,
+        "isTop3": False,
+        "isPremium": False,
+    }
+    return offer
+
+
+# ── Tests ────────────────────────────────────────────────────────────────────
+
+
+def test_cian_serp_parses_full_offer():
+    """Базовый тест: полный offer → ScrapedLot с ключевыми полями."""
+    offer = _make_full_offer()
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+
+    assert len(lots) == 1
+    lot = lots[0]
+
+    assert lot.source == "cian"
+    assert lot.source_id == "328739308"
+    assert lot.source_url == "https://ekb.cian.ru/sale/flat/328739308/"
+
+
+def test_cian_serp_coords_no_jitter():
+    """Координаты берутся точно из state — NO anchor jitter."""
+    offer = _make_full_offer(lat=56.8385, lng=60.5940)
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+
+    assert len(lots) == 1
+    lot = lots[0]
+    assert lot.lat == pytest.approx(56.8385, abs=1e-9)
+    assert lot.lon == pytest.approx(60.5940, abs=1e-9)
+
+
+def test_cian_serp_kitchen_area_in_raw_payload():
+    """kitchen_area_m2 не в ScrapedLot, но сохраняется в raw_payload."""
+    offer = _make_full_offer(kitchen_area="11.5")
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+
+    lot = lots[0]
+    assert lot.raw_payload is not None
+    assert lot.raw_payload["kitchen_area_m2"] == "11.5"
+
+
+def test_cian_serp_description_minhash():
+    """description_minhash берётся из Cian state, если есть."""
+    offer = _make_full_offer(
+        description_minhash="cian_minhash_abc123",
+        description="Тест описания",
+    )
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+
+    lot = lots[0]
+    assert lot.description_minhash == "cian_minhash_abc123"
+
+
+def test_cian_serp_description_minhash_fallback():
+    """Если Cian не дал minhash — вычисляем SHA1 из description."""
+    offer = _make_full_offer(description_minhash="", description="Уникальное описание квартиры")
+    offer["descriptionMinhash"] = ""  # явно пусто
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+
+    lot = lots[0]
+    assert lot.description_minhash is not None
+    assert len(lot.description_minhash) == 32  # SHA1 hex [:32]
+
+
+def test_cian_serp_phones():
+    """phones — список dicts из state."""
+    offer = _make_full_offer(has_phones=True)
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+
+    lot = lots[0]
+    assert isinstance(lot.phones, list)
+    assert len(lot.phones) == 1
+    assert lot.phones[0]["countryCode"] == "7"
+    assert lot.phones[0]["number"] == "9012345678"
+
+
+def test_cian_serp_house_ext_id_newbuilding():
+    """house_ext_id заполнен если offer.newbuilding.id > 0."""
+    offer = _make_full_offer(newbuilding_id=54016)
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+
+    lot = lots[0]
+    assert lot.house_source == "cian_newbuilding"
+    assert lot.house_ext_id == "54016"
+    assert lot.listing_segment == "novostroyki"
+
+
+def test_cian_serp_vtorichka_no_house_ext_id():
+    """Вторичка (newbuilding.id == 0) — house_ext_id None."""
+    offer = _make_full_offer(newbuilding_id=0)
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+
+    lot = lots[0]
+    assert lot.house_source == "cian"
+    assert lot.house_ext_id is None
+    assert lot.listing_segment == "vtorichka"
+
+
+def test_cian_serp_metro_stations():
+    """metro_stations — список dicts с name/time/mode/line_color."""
+    offer = _make_full_offer()
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+
+    lot = lots[0]
+    assert isinstance(lot.metro_stations, list)
+    assert len(lot.metro_stations) == 1
+    station = lot.metro_stations[0]
+    assert station["name"] == "Геологическая"
+    assert station["time"] == 10
+    assert station["mode"] == "walk"
+    assert station["line_color"] == "FF0000"
+
+
+def test_cian_serp_cadastral_numbers():
+    """Кадастровые номера квартиры и здания."""
+    offer = _make_full_offer(
+        cadastral_number="66:41:0204016:1234",
+        building_cadastral_number="66:41:0204016:100",
+    )
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+
+    lot = lots[0]
+    assert lot.cadastral_number == "66:41:0204016:1234"
+    assert lot.building_cadastral_number == "66:41:0204016:100"
+    # kadastr_num (base field) тоже заполнен
+    assert lot.kadastr_num == "66:41:0204016:1234"
+
+
+def test_cian_serp_is_homeowner_seller():
+    """is_homeowner и is_pro_seller из offer."""
+    offer = _make_full_offer(is_homeowner=True, is_pro=False)
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+
+    lot = lots[0]
+    assert lot.is_homeowner is True
+    assert lot.is_pro_seller is False
+
+
+def test_cian_serp_bargain_and_sale_type():
+    """bargain_allowed и sale_type из bargainTerms."""
+    offer = _make_full_offer(bargain_allowed=True, sale_type="alternative")
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+
+    lot = lots[0]
+    assert lot.bargain_allowed is True
+    assert lot.sale_type == "alternative"
+
+
+def test_cian_serp_building_fields():
+    """Поля здания: year_built, house_type, total_floors."""
+    offer = _make_full_offer(year_built=1988, material_type="brick", floors_count=9)
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+
+    lot = lots[0]
+    assert lot.year_built == 1988
+    assert lot.house_type == "brick"
+    assert lot.total_floors == 9
+
+
+def test_cian_serp_living_area():
+    """living_area_m2 из offer.livingArea."""
+    offer = _make_full_offer(living_area="35.0")
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+
+    lot = lots[0]
+    assert lot.living_area_m2 == pytest.approx(35.0)
+
+
+def test_cian_serp_state_missing_returns_empty():
+    """Если state нет в HTML — возвращаем пустой список."""
+    html = "<html><body><p>no cian state here</p></body></html>"
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+    assert lots == []
+
+
+def test_cian_serp_skips_offer_without_price():
+    """Offer без цены — пропускаем."""
+    offer = _make_full_offer()
+    del offer["bargainTerms"]["priceRur"]
+    del offer["bargainTerms"]["price"]
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+    assert lots == []
+
+
+def test_cian_serp_multiple_offers():
+    """Несколько offers → список ScrapedLot."""
+    offers = [_make_full_offer(cian_id=100 + i) for i in range(5)]
+    html = _make_cian_serp_html(offers)
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+    assert len(lots) == 5
+    ids = [lot.source_id for lot in lots]
+    assert sorted(ids) == ["100", "101", "102", "103", "104"]
+
+
+def test_cian_serp_photo_urls():
+    """photo_urls — полные URL из offer.photos[].fullUrl."""
+    offer = _make_full_offer(cian_id=999)
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+
+    lot = lots[0]
+    assert len(lot.photo_urls) == 1
+    assert "cdn-cian.ru" in lot.photo_urls[0]
+    assert "999-1.jpg" in lot.photo_urls[0]
+
+
+def test_cian_serp_no_coords_does_not_crash():
+    """Если координаты отсутствуют — lot.lat/lon = None, не краш."""
+    offer = _make_full_offer()
+    del offer["geo"]["coordinates"]
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+
+    assert len(lots) == 1
+    assert lots[0].lat is None
+    assert lots[0].lon is None
+
+
+# ── Address formatter tests ──────────────────────────────────────────────────
+
+
+def test_format_address_standard():
+    """Стандартный geo.address[]."""
+    parts = [
+        {"type": "location", "name": "Свердловская область"},
+        {"type": "city", "name": "Екатеринбург"},
+        {"type": "district", "name": "Чкаловский"},
+        {"type": "street", "name": "улица Постовского"},
+        {"type": "house", "name": "17А"},
+        {"type": "metro", "name": "Геологическая"},
+    ]
+    result = _format_address(parts)
+    # location и metro пропускаем
+    assert "Свердловская" not in result
+    assert "Геологическая" not in result
+    assert "Екатеринбург" in result
+    assert "17А" in result
+
+
+def test_format_address_empty():
+    """Пустой address[] → fallback string."""
+    assert _format_address([]) == "Екатеринбург (Cian)"