diff --git a/tradein-mvp/backend/tests/test_scraper_kit_cian_golden_parity.py b/tradein-mvp/backend/tests/test_scraper_kit_cian_golden_parity.py
new file mode 100644
index 00000000..5ca9ace7
--- /dev/null
+++ b/tradein-mvp/backend/tests/test_scraper_kit_cian_golden_parity.py
@@ -0,0 +1,492 @@
+"""Golden-parity: `scraper_kit.providers.cian.*` парсинг ≡ `app.services.scrapers.cian*`.
+
+Strangler-инвариант (#2133 cian): новая scraper_kit-копия cian-провайдера должна давать
+БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе. Развязка
+(settings→ScraperConfig, get_scraper_delay→delay_provider, cian_session→локальная копия)
+НЕ меняет распарсенные данные.
+
+Гоняем ОБА модуля на одних фикстурах и сравниваем результат:
+ - SERP `_parse_serp_html` (Redux state → list[ScrapedLot]) на синтетическом _cianConfig
+ push-HTML (вторичка + новостройка + студия), сравнение model_dump();
+ - SERP `_offer_to_lot` пооферно + `_format_address` (pure, восстановление дома ЖК);
+ - `_extract_total_offers` (totalOffers из state);
+ - `extract_state` — app-версия ≡ kit-версия на одном HTML (общий парсер state);
+ - detail `fetch_detail` (defaultState + bti sister → DetailEnrichment, dataclasses.asdict);
+ - valuation `_parse_valuation_state` (estimation/chart/houseInfo → CianValuationResult);
+ - newbuilding pure-экстракторы (_extract_chart / _extract_reliability_checks /
+ _extract_nested_offers / _extract_zhk_url_from_serp).
+
+Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import os
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+# Старый app.services.scrapers.cian* импортирует app.core.config.settings=Settings(),
+# которому нужен DATABASE_URL. Офлайн-парсинг БД не трогает — фиктивный DSN достаточен.
+os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
+
+# НОВЫЙ (scraper_kit) cian-провайдер.
+from scraper_kit.cian_state_parser import extract_state as new_extract_state
+from scraper_kit.providers.cian.detail import fetch_detail as new_fetch_detail
+from scraper_kit.providers.cian.newbuilding import _extract_chart as new_extract_chart
+from scraper_kit.providers.cian.newbuilding import (
+ _extract_nested_offers as new_extract_nested_offers,
+)
+from scraper_kit.providers.cian.newbuilding import (
+ _extract_reliability_checks as new_extract_reliability,
+)
+from scraper_kit.providers.cian.newbuilding import (
+ _extract_zhk_url_from_serp as new_extract_zhk_url,
+)
+from scraper_kit.providers.cian.serp import CianScraper as NewCianScraper
+from scraper_kit.providers.cian.serp import _format_address as new_format_address
+from scraper_kit.providers.cian.valuation import (
+ _parse_valuation_state as new_parse_valuation,
+)
+
+# СТАРЫЙ (app.services.scrapers) cian-провайдер.
+from app.services.scrapers.cian import CianScraper as OldCianScraper
+from app.services.scrapers.cian import _format_address as old_format_address
+from app.services.scrapers.cian_detail import fetch_detail as old_fetch_detail
+from app.services.scrapers.cian_newbuilding import _extract_chart as old_extract_chart
+from app.services.scrapers.cian_newbuilding import (
+ _extract_nested_offers as old_extract_nested_offers,
+)
+from app.services.scrapers.cian_newbuilding import (
+ _extract_reliability_checks as old_extract_reliability,
+)
+from app.services.scrapers.cian_newbuilding import (
+ _extract_zhk_url_from_serp as old_extract_zhk_url,
+)
+from app.services.scrapers.cian_state_parser import extract_state as old_extract_state
+from app.services.scrapers.cian_valuation import (
+ _parse_valuation_state as old_parse_valuation,
+)
+
+# ── Fixtures: SERP offers → _cianConfig push HTML ────────────────────────────
+
+_CONFIG = SimpleNamespace(glitchtip_dsn=None)
+
+
+def _serp_html(offers: list[dict[str, Any]]) -> str:
+ """Обернуть offers в валидный _cianConfig['frontend-serp'].push() HTML."""
+ state = {"results": {"totalOffers": len(offers), "offers": offers}}
+ state_json = json.dumps(state, ensure_ascii=False)
+ return (
+ ""
+ )
+
+
+def _vtorichka_offer() -> dict[str, Any]:
+ return {
+ "cianId": 328739308,
+ "id": 328739308,
+ "fullUrl": "https://ekb.cian.ru/sale/flat/328739308/",
+ "offerType": "flat",
+ "flatType": "rooms",
+ "category": "flatSale",
+ "isApartments": False,
+ "roomsCount": 2,
+ "totalArea": "52.0",
+ "livingArea": "35.0",
+ "kitchenArea": "11.5",
+ "balconiesCount": 1,
+ "loggiasCount": 0,
+ "bedroomsCount": None,
+ "hasFurniture": False,
+ "isSoldFurnished": False,
+ "decoration": "euro",
+ "floorNumber": 7,
+ "building": {
+ "floorsCount": 16,
+ "materialType": "monolithBrick",
+ "buildYear": 2010,
+ "parking": {"type": "underground"},
+ "totalArea": "3400",
+ },
+ "cadastralNumber": "66:41:0204016:1234",
+ "buildingCadastralNumber": "66:41:0204016:100",
+ "bargainTerms": {
+ "priceRur": 6500000,
+ "price": 6500000,
+ "saleType": "free",
+ "mortgageAllowed": True,
+ "bargainAllowed": True,
+ },
+ "geo": {
+ "coordinates": {"lat": 56.8385, "lng": 60.5940},
+ "address": [
+ {"type": "location", "fullName": "Свердловская область"},
+ {"type": "raion", "fullName": "Кировский"},
+ {"type": "street", "fullName": "улица Постовского"},
+ {"type": "house", "fullName": "17А"},
+ {"type": "metro", "fullName": "Геологическая"},
+ ],
+ "undergrounds": [
+ {
+ "name": "Геологическая",
+ "time": 10,
+ "transportType": "walk",
+ "lineId": 10,
+ "lineColor": "FF0000",
+ "isDefault": True,
+ }
+ ],
+ },
+ "newbuilding": {"id": 0, "name": ""},
+ "phones": [{"number": "9012345678", "countryCode": "7"}],
+ "userId": 13332612,
+ "publishedUserId": 13332612,
+ "isByHomeowner": True,
+ "isPro": False,
+ "isRosreestrChecked": True,
+ "description": "Продаётся отличная квартира с евроремонтом",
+ "descriptionMinhash": "abc123def456abc1",
+ "addedTimestamp": 1716451200,
+ "photos": [{"fullUrl": "https://images.cdn-cian.ru/images/328739308-1.jpg"}],
+ }
+
+
+def _newbuilding_offer() -> dict[str, Any]:
+ """Новостройка без части house в geo.address[] — жмёт _recover_newbuilding_house."""
+ return {
+ "cianId": 411112222,
+ "id": 411112222,
+ "fullUrl": "https://ekb.cian.ru/sale/flat/411112222/",
+ "offerType": "flat",
+ "flatType": "rooms",
+ "isApartments": False,
+ "roomsCount": 1,
+ "totalArea": "38.0",
+ "floorNumber": 12,
+ "building": {
+ "floorsCount": 25,
+ "materialType": "monolith",
+ "buildYear": None,
+ "deadline": {"year": 2026},
+ },
+ "buildingCadastralNumber": "66:41:0204016:11396",
+ "bargainTerms": {"priceRur": 5200000, "price": 5200000, "mortgageAllowed": True},
+ "geo": {
+ "coordinates": {"lat": 56.8000, "lng": 60.6000},
+ "address": [
+ {"type": "location", "fullName": "Свердловская область"},
+ {"type": "street", "fullName": "улица Новая"},
+ ],
+ },
+ "newbuilding": {"id": 48853, "name": "ЖК Парковый квартал 29А"},
+ "isFromDeveloper": True,
+ "photos": [],
+ }
+
+
+def _studio_offer() -> dict[str, Any]:
+ return {
+ "cianId": 500011111,
+ "id": 500011111,
+ "offerType": "flat",
+ "flatType": "studio",
+ "roomsCount": None,
+ "totalArea": "24.5",
+ "floorNumber": 3,
+ "building": {"floorsCount": 9, "materialType": "panel", "buildYear": 1980},
+ "bargainTerms": {"priceRur": 3100000, "price": 3100000},
+ "geo": {
+ "coordinates": {"lat": 56.81, "lng": 60.61},
+ "address": [
+ {"type": "street", "fullName": "улица Ленина"},
+ {"type": "house", "fullName": "5"},
+ ],
+ },
+ "newbuilding": {"id": 0},
+ "photos": [],
+ }
+
+
+_OFFERS = [_vtorichka_offer(), _newbuilding_offer(), _studio_offer()]
+
+
+def _make_old() -> OldCianScraper:
+ with patch("app.services.scrapers.cian.get_scraper_delay", return_value=5.0):
+ return OldCianScraper()
+
+
+def _make_new() -> NewCianScraper:
+ return NewCianScraper(config=_CONFIG)
+
+
+def _dump(lots: list[Any]) -> list[dict[str, Any]]:
+ return [lot.model_dump() for lot in lots]
+
+
+# ── SERP _parse_serp_html parity ─────────────────────────────────────────────
+
+
+def test_serp_parse_serp_html_parity() -> None:
+ """Redux offers → list[ScrapedLot] идентичны (вторичка + новостройка + студия)."""
+ html = _serp_html(_OFFERS)
+ old_lots = _make_old()._parse_serp_html(html)
+ new_lots = _make_new()._parse_serp_html(html)
+
+ assert len(old_lots) == len(new_lots) == 3
+ assert _dump(old_lots) == _dump(new_lots)
+
+
+def test_serp_offer_to_lot_parity_per_offer() -> None:
+ """Пооферный _offer_to_lot: model_dump идентичен для каждого offer."""
+ old, new = _make_old(), _make_new()
+ for offer in _OFFERS:
+ old_lot = old._offer_to_lot(offer)
+ new_lot = new._offer_to_lot(offer)
+ assert (old_lot is None) == (new_lot is None)
+ if old_lot is not None and new_lot is not None:
+ assert old_lot.model_dump() == new_lot.model_dump()
+
+
+def test_serp_extract_total_offers_parity() -> None:
+ html = _serp_html(_OFFERS)
+ assert _make_old()._extract_total_offers(html) == _make_new()._extract_total_offers(html)
+
+
+def test_format_address_parity() -> None:
+ """_format_address (pure) — вкл. восстановление дома для ЖК без части house."""
+ building = {"houseNumber": None}
+ nb = {"name": "ЖК Парковый квартал 29А к1"}
+ cases = [
+ # (address_parts, is_newbuilding kwargs)
+ ([], {}),
+ (
+ [
+ {"type": "location", "fullName": "Свердловская область"},
+ {"type": "street", "fullName": "улица Мира"},
+ {"type": "house", "fullName": "12"},
+ ],
+ {},
+ ),
+ (
+ [
+ {"type": "location", "fullName": "Свердловская область"},
+ {"type": "street", "fullName": "улица Новая"},
+ ],
+ {
+ "building": building,
+ "building_cadastral_number": "66:41:0204016:11396",
+ "newbuilding": nb,
+ "is_newbuilding": True,
+ },
+ ),
+ ]
+ for parts, kwargs in cases:
+ assert old_format_address(parts, **kwargs) == new_format_address(parts, **kwargs)
+
+
+def test_extract_state_parity() -> None:
+ """app.extract_state ≡ scraper_kit.extract_state на SERP-фикстуре."""
+ html = _serp_html(_OFFERS)
+ old_state = old_extract_state(html, mfe="frontend-serp", key="initialState")
+ new_state = new_extract_state(html, mfe="frontend-serp", key="initialState")
+ assert old_state == new_state
+ assert new_state is not None
+
+
+# ── detail fetch_detail parity ───────────────────────────────────────────────
+
+
+def _detail_html() -> str:
+ """defaultState + bti sister → _cianConfig['frontend-offer-card'] push HTML."""
+ default_state = {
+ "offerData": {
+ "offer": {
+ "cianId": 327830237,
+ "windowsViewType": "yardAndStreet",
+ "separateWcsCount": 1,
+ "combinedWcsCount": 0,
+ "building": {"ceilingHeight": 2.65},
+ "repairType": "cosmetic",
+ "kitchenArea": "11.5",
+ "description": "Отличная квартира",
+ "priceChanges": [
+ {"changeTime": "2026-04-01T00:00:00Z", "price": 5500000, "diffPercent": -1.8}
+ ],
+ "agent": {
+ "id": 7654321,
+ "companyName": "Первичка Плюс",
+ "isPro": True,
+ "skills": [],
+ "accountType": "specialist",
+ },
+ "newbuilding": {"id": 0},
+ },
+ "stats": {
+ "totalViewsFormattedString": "1 042",
+ "todayViewsFormattedString": "7",
+ },
+ }
+ }
+ bti_state = {"houseData": {"series": "1-464", "wallMaterial": "brick"}}
+ ds_json = json.dumps(default_state, ensure_ascii=False)
+ bti_json = json.dumps(bti_state, ensure_ascii=False)
+ return (
+ ""
+ )
+
+
+def _mock_session(html: str) -> MagicMock:
+ response = MagicMock()
+ response.status_code = 200
+ response.text = html
+ session = MagicMock()
+ session.get = AsyncMock(return_value=response)
+ session.close = AsyncMock()
+ return session
+
+
+@pytest.mark.asyncio
+async def test_detail_fetch_parity() -> None:
+ """fetch_detail (defaultState + bti) → DetailEnrichment идентичен (все поля)."""
+ html = _detail_html()
+ url = "https://ekb.cian.ru/sale/flat/327830237/"
+
+ old_res = await old_fetch_detail(url, session=_mock_session(html))
+ new_res = await new_fetch_detail(url, session=_mock_session(html))
+
+ assert old_res is not None
+ assert new_res is not None
+ assert dataclasses.asdict(old_res) == dataclasses.asdict(new_res)
+
+
+# ── valuation _parse_valuation_state parity ──────────────────────────────────
+
+
+def _valuation_state() -> dict[str, Any]:
+ return {
+ "user": {"isAuthenticated": True, "userId": 42},
+ "estimation": {
+ "sale": {
+ "data": {
+ "price": 6800000,
+ "accuracy": 87,
+ "priceFrom": 6400000,
+ "priceTo": 7200000,
+ "priceSqm": 130000,
+ "filtersHash": "hash-abc",
+ }
+ },
+ "rent": {
+ "data": {
+ "price": 42000,
+ "accuracy": 80,
+ "priceFrom": 39000,
+ "priceTo": 45000,
+ "taxPrice": 5460,
+ }
+ },
+ },
+ "estimationChart": {
+ "data": {
+ "title": {"change": "increase", "changeValue": "8,3%"},
+ "chartData": {
+ "data": [
+ {"date": 1714521600000, "price": 6700000, "priceFormatted": "6,7 млн"},
+ {"date": 1717200000000, "price": 6800000, "priceFormatted": "6,8 млн"},
+ ]
+ },
+ }
+ },
+ "houseInfo": {
+ "data": {
+ "items": [
+ {"title": "Год постройки", "value": "2010"},
+ {"title": "Тип дома", "value": "монолит"},
+ ]
+ }
+ },
+ "filters": {"houseId": 998877},
+ "managementCompany": {"data": {"name": "УК Ромашка", "phones": ["+73430000000"]}},
+ }
+
+
+def test_valuation_parse_state_parity() -> None:
+ """_parse_valuation_state → CianValuationResult идентичен (все поля)."""
+ state = _valuation_state()
+ old_res = old_parse_valuation(state)
+ new_res = new_parse_valuation(state)
+ assert dataclasses.asdict(old_res) == dataclasses.asdict(new_res)
+
+
+# ── newbuilding pure-экстракторы parity ──────────────────────────────────────
+
+
+def test_newbuilding_extractors_parity() -> None:
+ rv_state = {
+ "data": {
+ "priceDynamics": {
+ "chart": {
+ "data": {"labels": ["2026-01-01", "2026-02-01"], "values": [120000, 125000]}
+ }
+ }
+ }
+ }
+ assert old_extract_chart(rv_state) == new_extract_chart(rv_state)
+
+ rel_state = {
+ "checkStatus": {"status": "reliable", "title": "Надёжный застройщик"},
+ "details": [{"title": "Проектная декларация", "type": "doc"}],
+ }
+ assert old_extract_reliability(rel_state) == new_extract_reliability(rel_state)
+
+ offers_state = {
+ "data": {
+ "total": {
+ "fromDeveloperRooms": [
+ {"roomType": "1", "layouts": [{"id": 1, "area": 38}]},
+ {"roomType": "2", "layouts": [{"id": 2, "area": 55}]},
+ ]
+ }
+ }
+ }
+ assert old_extract_nested_offers(offers_state) == new_extract_nested_offers(offers_state)
+
+ serp_html = (
+ ''
+ )
+ assert old_extract_zhk_url(serp_html) == new_extract_zhk_url(serp_html)
+
+
+# ── strangler-guard: kit-провайдер не импортирует app.* ──────────────────────
+
+
+def test_kit_cian_has_no_app_imports() -> None:
+ """Ни один модуль scraper_kit.providers.cian.* не импортирует app.* (развязка)."""
+ import inspect
+
+ from scraper_kit.providers.cian import detail, newbuilding, serp, session, valuation
+
+ for mod in (serp, detail, newbuilding, valuation, session):
+ source = inspect.getsource(mod)
+ for line in source.splitlines():
+ stripped = line.strip()
+ assert not stripped.startswith("from app"), f"{mod.__name__}: {line!r}"
+ assert not stripped.startswith("import app"), f"{mod.__name__}: {line!r}"
diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/contracts.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/contracts.py
index eb04dc5e..de6f5d54 100644
--- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/contracts.py
+++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/contracts.py
@@ -108,6 +108,8 @@ class ScraperConfig(Protocol):
# Границы валидности cian-оценки, руб.
cian_valuation_min_rub: float
cian_valuation_max_rub: float
+ # Секрет pgp_sym_encrypt для cian_session_cookies (read в session.py).
+ cookie_encryption_key: str
# Sentry/GlitchTip DSN (опционально).
glitchtip_dsn: str | None
diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/house_type_normalizer.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/house_type_normalizer.py
new file mode 100644
index 00000000..f9ad280d
--- /dev/null
+++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/house_type_normalizer.py
@@ -0,0 +1,90 @@
+"""Нормализация house_type → каноничный enum для listings.house_type.
+
+Целевой канон (сверено с estimator._IMV_HOUSE_TYPE_MAP, estimator.py:146):
+ panel / brick / monolith / monolith_brick / block / wood.
+
+Зачем (#2007): estimator применяет soft-penalty по house_type — аналог с
+house_type != target штрафуется. Yandex SERP отдаёт SCREAMING-значения
+(MONOLIT / BRICK / PANEL / ...), которые НИКОГДА не равны каноничным
+(monolith / brick / panel / ...) → ~70% yandex-аналогов получали ложный штраф
+и фактически выпадали из скоринга. Нормализация на ингесте чинит это.
+
+Важно про None: неизвестное / 'other' / '' → None, НЕ 'other'. В estimator
+`house_type IS NULL` нейтрально (без штрафа), а любое не-канон значение всегда
+!= target → ложный штраф. Поэтому unknown лучше схлопнуть в NULL.
+
+Источники raw-значений:
+- yandex SERP: building.buildingType — SCREAMING_SNAKE (MONOLIT, MONOLIT_BRICK, ...)
+- cian SERP: building.materialType — camelCase (monolith, monolithBrick,
+ gasSilicateBlock, ...) — переиспользуется в #2008 (cian.py:815).
+"""
+
+from __future__ import annotations
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+# Канон listings.house_type — выровнен по estimator._IMV_HOUSE_TYPE_MAP.
+_CANON: frozenset[str] = frozenset(
+ {"panel", "brick", "monolith", "monolith_brick", "block", "wood"}
+)
+
+# raw-токен → канон. Ключи — точные значения вокабуляров источников; сравнение
+# регистронезависимое (см. _LOOKUP ниже), но строго по полному токену енума,
+# без fuzzy-матчинга. Неизвестные токены сюда НЕ попадают → normalize вернёт None.
+_RAW_TO_CANON: dict[str, str] = {
+ # ── yandex SERP (SCREAMING_SNAKE buildingType) ───────────────────────────
+ "MONOLIT": "monolith",
+ "BRICK": "brick",
+ "PANEL": "panel",
+ "MONOLIT_BRICK": "monolith_brick",
+ "BLOCK": "block",
+ "WOOD": "wood",
+ # ── cian SERP (camelCase materialType) — reuse в #2008 ───────────────────
+ "monolith": "monolith",
+ "brick": "brick",
+ "panel": "panel",
+ "block": "block",
+ "wood": "wood",
+ "monolithBrick": "monolith_brick",
+ "gasSilicateBlock": "block",
+ "aerocreteBlock": "block",
+ "foamConcreteBlock": "block",
+ "stalin": "brick", # «сталинка» — кирпич
+}
+
+# Регистронезависимый lookup. Лоуэркейс-ключи не коллизят между вокабулярами:
+# 'MONOLIT'→'monolit' и 'monolith'→'monolith' — разные ключи.
+_LOOKUP: dict[str, str] = {k.lower(): v for k, v in _RAW_TO_CANON.items()}
+
+
+def normalize_house_type(raw: str | None) -> str | None:
+ """Преобразовать raw house_type в каноничный enum.
+
+ Уже-каноничное значение возвращается as-is (идемпотентность — нужна при
+ ре-обработке и для backfill-миграции). Неизвестное / 'other' / пустое /
+ None → None (НЕ 'other': см. docstring модуля про estimator soft-penalty).
+
+ Args:
+ raw: сырое значение из парсера (e.g. «MONOLIT», «monolithBrick») или None.
+
+ Returns:
+ Одно из panel / brick / monolith / monolith_brick / block / wood, либо None.
+ """
+ if raw is None:
+ return None
+ stripped = raw.strip()
+ if not stripped:
+ return None
+ # Pass-through: уже каноничное значение (idempotency — в т.ч. 'monolith_brick',
+ # которого нет среди raw-ключей карты).
+ if stripped in _CANON:
+ return stripped
+ # Регистронезависимый exact-token lookup по обоим вокабулярам.
+ canon = _LOOKUP.get(stripped) or _LOOKUP.get(stripped.lower())
+ if canon is None:
+ # Неизвестное / 'other' — нормально (NULL нейтрально для estimator).
+ # debug, не warning: 'other' встречается массово, warning засорил бы лог.
+ logger.debug("house_type_normalizer: unmapped raw value %r — stored as NULL", raw)
+ return canon
diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/__init__.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/__init__.py
new file mode 100644
index 00000000..7c2c82b2
--- /dev/null
+++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/__init__.py
@@ -0,0 +1,17 @@
+"""Cian-провайдер scraper_kit — strangler-копия `app.services.scrapers.cian*`.
+
+Модули:
+ - serp — SERP-sweep ядро (CianScraper, Redux state → ScrapedLot, price-bisect)
+ - detail — detail-страница объявления (fetch_detail → DetailEnrichment)
+ - newbuilding — каталог ЖК (fetch_newbuilding → NewbuildingEnrichment, resolve zhk-url)
+ - valuation — Cian Valuation Calculator (estimate_via_cian_valuation)
+ - session — auth cookie management (load/save/verify/mark-invalid)
+
+Развязка от `app.*`:
+ - `app.core.config.settings` → инжектируемый `ScraperConfig` (scraper_kit.contracts)
+ - `app.services.scraper_settings.get_scraper_delay` → инжектируемый `delay_provider`
+ - `app.services.scrapers.cian_state_parser` → `scraper_kit.cian_state_parser`
+ - `app.services.scrapers.{base,browser_fetcher,price_brackets,repair_state_normalizer,
+ snapshot_writer,house_type_normalizer}` → `scraper_kit.*`
+ - `app.services.cian_session` → локальная копия `scraper_kit.providers.cian.session`
+"""
diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/detail.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/detail.py
new file mode 100644
index 00000000..0152cdd0
--- /dev/null
+++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/detail.py
@@ -0,0 +1,450 @@
+"""Cian.ru detail page scraper — single offer enrichment.
+
+Different from SERP (cian.py):
+- URL: https://ekb.cian.ru/sale/flat// or https://www.cian.ru/sale/flat//
+- MFE: 'frontend-offer-card'
+- State KEY: 'defaultState' (NOT 'initialState' — SERP uses initialState)
+- Sister containers in same _cianConfig: bti, priceChanges, stats, agent, newObject
+
+Parses ~88 offer fields + sister data → DetailEnrichment dataclass.
+Stage 5 of CianScraper v1.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+from curl_cffi.requests import AsyncSession
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from scraper_kit.cian_state_parser import extract_all_states, extract_state
+from scraper_kit.repair_state_normalizer import (
+ infer_repair_state_from_text,
+ normalize_repair_state,
+)
+from scraper_kit.snapshot_writer import upsert_listing_snapshot
+
+if TYPE_CHECKING:
+ from scraper_kit.browser_fetcher import BrowserFetcher
+ from scraper_kit.contracts import ScraperConfig
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class DetailEnrichment:
+ """Result of cian_detail.fetch_detail() — fields для UPDATE listings + sister rows."""
+
+ # Core listing enrichment (extend existing listings row)
+ cian_id: int | None = None
+ windows_view_type: str | None = None # 'yard' / 'street' / 'yardAndStreet'
+ separate_wcs_count: int | None = None
+ combined_wcs_count: int | None = None
+ ceiling_height: float | None = None # meters — offer.building.ceilingHeight (#1791)
+ repair_type: str | None = None # raw Cian: offer.repairType OR offer.decoration (#1791)
+ repair_state: str | None = None # enum: needs_repair/standard/good/excellent
+ kitchen_area_m2: float | None = None # offer.kitchenArea (м²)
+ description: str | None = None # offer.description (текст объявления)
+ views_total: int | None = None # Cian stats.totalViewsFormattedString → int
+ views_today: int | None = None
+
+ # BTI (вторичка only — primary buildings don't have БТИ)
+ bti_data: dict[str, Any] | None = None # raw bti.houseData snapshot
+
+ # Price history (offer.priceChanges)
+ price_changes: list[dict[str, Any]] = field(default_factory=list)
+
+ # Agent profile (offer.agent.* expanded)
+ agent_profile: dict[str, Any] | None = None
+
+ # Newbuilding link (if offer is from a newbuilding)
+ newbuilding_data: dict[str, Any] | None = None
+
+ # Raw payload для debugging / future-extraction
+ raw_offer: dict[str, Any] | None = None
+ raw_sister_states: dict[str, Any] = field(default_factory=dict)
+
+
+async def fetch_detail(
+ offer_url: str,
+ *,
+ config: ScraperConfig | None = None,
+ session: AsyncSession | None = None,
+ browser_fetcher: BrowserFetcher | None = None,
+) -> DetailEnrichment | None:
+ """Fetch detail page and extract all containers.
+
+ Args:
+ offer_url: e.g. 'https://ekb.cian.ru/sale/flat/327830237/'
+ config: инжектируемый ScraperConfig (#2133) — читается только на curl_cffi-пути
+ для прокси-egress (config.cian_proxy_url). None ⇒ прямое подключение
+ (dev / browser_fetcher-путь, где прокси уже применён server-side).
+ session: optional shared curl_cffi session (для batched scrapes).
+ Ignored when browser_fetcher is provided.
+ browser_fetcher: optional BrowserFetcher instance (camoufox HTTP service).
+ When provided, fetches JS-rendered HTML via the browser service instead of
+ curl_cffi — enables priceChanges extraction which requires JS rendering.
+ Caller is responsible for the context-manager lifecycle of the fetcher.
+
+ Returns: DetailEnrichment, or None если fetch / parse failed.
+ """
+ if browser_fetcher is not None:
+ # Browser path: get fully JS-rendered HTML; same parse path follows.
+ try:
+ html = await browser_fetcher.fetch(offer_url)
+ except Exception as exc:
+ logger.warning("Cian detail browser fetch failed %s: %s", offer_url, exc)
+ return None
+ else:
+ # curl_cffi path (legacy, back-compat).
+ close_session = False
+ if session is None:
+ # proxies: mobile-proxy egress (#806) — без прокси datacenter-IP блокируется Cian.
+ # Пусто (config не передан / env не задан) → прямое подключение (dev/no-op).
+ _proxy_url = config.cian_proxy_url if config is not None else None
+ _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
+ session = AsyncSession(
+ impersonate="chrome120",
+ timeout=25.0,
+ proxies=_proxies,
+ headers={
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
+ },
+ )
+ close_session = True
+
+ try:
+ resp = await session.get(offer_url, allow_redirects=True)
+ if resp.status_code != 200:
+ logger.warning("Cian detail fetch %s → HTTP %d", offer_url, resp.status_code)
+ return None
+ html = resp.text
+ finally:
+ if close_session:
+ await session.close()
+
+ # ── Shared parse path (identical regardless of HTML source) ──────────────
+
+ # Primary state container — defaultState in frontend-offer-card MFE
+ # NOTE: detail pages use 'defaultState', SERP uses 'initialState'
+ offer_state = extract_state(html, mfe="frontend-offer-card", key="defaultState")
+ if offer_state is None:
+ logger.warning("Cian detail %s: defaultState extraction failed", offer_url)
+ return None
+
+ offer_data = offer_state.get("offerData", {})
+ offer = offer_data.get("offer", {})
+
+ result = DetailEnrichment(
+ cian_id=offer.get("cianId") or offer.get("id"),
+ windows_view_type=_extract_windows_view(offer),
+ separate_wcs_count=offer.get("separateWcsCount"),
+ combined_wcs_count=offer.get("combinedWcsCount"),
+ ceiling_height=_extract_ceiling_height(offer),
+ repair_type=_extract_repair_type(offer),
+ repair_state=_extract_repair_state(offer),
+ kitchen_area_m2=_parse_float(offer.get("kitchenArea")),
+ description=offer.get("description") or None,
+ raw_offer=offer,
+ )
+
+ # Stats (views) — from stats key in offerData or top-level state
+ stats = offer_data.get("stats") or offer_state.get("stats") or {}
+ result.views_total = _parse_views(stats.get("totalViewsFormattedString"))
+ result.views_today = _parse_views(stats.get("todayViewsFormattedString"))
+
+ # Price changes — browser HTML: offerData.priceChanges (priceData.price format)
+ # curl_cffi HTML: offer.priceChanges or state.priceChanges (flat price field)
+ result.price_changes = _extract_price_changes(offer, offer_data, offer_state)
+
+ # Agent profile expansion
+ result.agent_profile = _extract_agent_profile(offer)
+
+ # Sister state containers: bti, newObject, plus raw stash of everything else
+ all_states = extract_all_states(html)
+ offer_card_states = all_states.get("frontend-offer-card", {})
+
+ bti_state = offer_card_states.get("bti")
+ if bti_state:
+ result.bti_data = bti_state.get("houseData")
+ result.raw_sister_states["bti"] = bti_state
+
+ # Newbuilding link (if applicable)
+ newbuilding = offer.get("newbuilding")
+ if newbuilding and newbuilding.get("id"):
+ result.newbuilding_data = newbuilding
+
+ # Stash all non-defaultState sister containers for debugging / future-extraction
+ for k, v in offer_card_states.items():
+ if k != "defaultState":
+ result.raw_sister_states[k] = v
+
+ return result
+
+
+# ── Field extractors ──────────────────────────────────────────────────────────
+
+
+def _extract_windows_view(offer: dict[str, Any]) -> str | None:
+ return offer.get("windowsViewType")
+
+
+def _extract_ceiling_height(offer: dict[str, Any]) -> float | None:
+ """Высота потолков (м).
+
+ На detail-странице (defaultState) значение лежит в ``offer.building.ceilingHeight``,
+ а НЕ в ``offer.ceilingHeight`` (последнее всегда null) — #1791. Top-level оставлен
+ как fallback на случай иной структуры карточки.
+ """
+ building = offer.get("building") or {}
+ return _parse_float(building.get("ceilingHeight") or offer.get("ceilingHeight"))
+
+
+def _extract_repair_type(offer: dict[str, Any]) -> str | None:
+ # Cian: 'cosmetic' / 'design' / 'no' / 'euro' / ...
+ # На detail-странице структурное поле чаще приходит как `decoration`
+ # ('preFine'/'fine'/...), а `repairType` пустое (#1791) — читаем оба.
+ return offer.get("repairType") or offer.get("decoration")
+
+
+def _extract_repair_state(offer: dict[str, Any]) -> str | None:
+ """Нормализованный enum из repairType/decoration → needs_repair/standard/good/excellent.
+
+ Если структурное поле пустое — fallback на инференс из описания (#622).
+ Структурное значение всегда в приоритете.
+ """
+ state = normalize_repair_state(offer.get("repairType") or offer.get("decoration"))
+ if state is None:
+ state = infer_repair_state_from_text(offer.get("description"))
+ return state
+
+
+def _parse_float(value: Any) -> float | None:
+ if value is None:
+ return None
+ try:
+ return float(value)
+ except (ValueError, TypeError):
+ return None
+
+
+def _parse_views(formatted_str: str | None) -> int | None:
+ """Cian's stats.totalViewsFormattedString — '1 234' → 1234."""
+ if not formatted_str:
+ return None
+ try:
+ return int(str(formatted_str).replace(" ", "").replace(",", ""))
+ except (ValueError, TypeError):
+ return None
+
+
+def _extract_price_changes(
+ offer: dict[str, Any],
+ offer_data: dict[str, Any],
+ state: dict[str, Any],
+) -> list[dict[str, Any]]:
+ """Normalize priceChanges from any of the known Cian locations/formats.
+
+ Lookup priority:
+ 1. offer.priceChanges — curl_cffi flat format: {changeTime, price, diffPercent}
+ 2. offerData.priceChanges — browser-rendered format: {changeTime, priceData:{price}}
+ 3. state.priceChanges — top-level fallback (rare)
+
+ Normalizes to: {change_time, price_rub, diff_percent}.
+ """
+ raw = (
+ offer.get("priceChanges")
+ or offer_data.get("priceChanges")
+ or state.get("priceChanges")
+ or []
+ )
+ if not isinstance(raw, list):
+ return []
+ result = []
+ for entry in raw:
+ if not isinstance(entry, dict):
+ continue
+ # price_rub: flat field (curl_cffi) OR nested priceData.price (browser)
+ price_data = entry.get("priceData") or {}
+ price_rub = entry.get("price") or entry.get("priceRub") or price_data.get("price")
+ result.append(
+ {
+ "change_time": entry.get("changeTime") or entry.get("date"),
+ "price_rub": price_rub,
+ "diff_percent": entry.get("diffPercent") or entry.get("diff_percent"),
+ }
+ )
+ return result
+
+
+def _extract_agent_profile(offer: dict[str, Any]) -> dict[str, Any] | None:
+ """offer.agent or offer.agency or offer.user — extract canonical profile shape."""
+ agent = offer.get("agent") or {}
+ user = offer.get("user") or {}
+ if not agent and not user:
+ return None
+ return {
+ "ext_agent_id": agent.get("id") or user.get("cianUserId"),
+ "company_name": agent.get("companyName") or user.get("companyName"),
+ "is_pro": agent.get("isPro") or user.get("isPro"),
+ "skills": agent.get("skills") or [],
+ "account_type": agent.get("accountType") or user.get("accountType"),
+ }
+
+
+# ── Save helpers ──────────────────────────────────────────────────────────────
+
+
+def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnrichment) -> None:
+ """Persist DetailEnrichment to DB:
+
+ - UPDATE listings SET windows_view_type, ceiling_height, ... WHERE id = listing_id
+ - INSERT INTO offer_price_history rows (if price_changes non-empty)
+ - INSERT INTO agents (if agent_profile) and link via agent_id_fk
+ - bti_data не сохраняется здесь — это задача Stage 6 (houses), где есть house_id
+
+ Idempotency: re-running безопасно (UPSERT / ON CONFLICT DO NOTHING semantics).
+ """
+ # Extend listings row with detail-enriched fields
+ db.execute(
+ text("""
+ UPDATE listings SET
+ windows_view_type = COALESCE(:wvt, windows_view_type),
+ separate_wcs_count = COALESCE(:swc, separate_wcs_count),
+ combined_wcs_count = COALESCE(:cwc, combined_wcs_count),
+ ceiling_height = COALESCE(:ch, ceiling_height),
+ repair_type = COALESCE(:rt, repair_type),
+ repair_state = COALESCE(:rs, repair_state),
+ kitchen_area_m2 = COALESCE(CAST(:ka AS double precision), kitchen_area_m2),
+ description = COALESCE(:descr, description),
+ views_total = COALESCE(:vt, views_total),
+ views_today = COALESCE(:vd, views_today),
+ detail_enriched_at = NOW()
+ WHERE id = CAST(:lid AS bigint)
+ """),
+ {
+ "lid": listing_id,
+ "wvt": enrichment.windows_view_type,
+ "swc": enrichment.separate_wcs_count,
+ "cwc": enrichment.combined_wcs_count,
+ "ch": enrichment.ceiling_height,
+ "rt": enrichment.repair_type,
+ "rs": enrichment.repair_state,
+ "ka": enrichment.kitchen_area_m2,
+ "descr": enrichment.description,
+ "vt": enrichment.views_total,
+ "vd": enrichment.views_today,
+ },
+ )
+
+ # Snapshot: point-in-time observation в listings_snapshots.
+ # Берём price_rub / price_per_m2 из listings строки (только что обновлённой).
+ # Используем COALESCE — на случай если listings.price_rub NULL (не должно быть, но safe).
+ _snap_row = db.execute(
+ text("""
+ SELECT price_rub, price_per_m2
+ FROM listings
+ WHERE id = CAST(:lid AS bigint)
+ """),
+ {"lid": listing_id},
+ ).fetchone()
+ _snap_written = False
+ if _snap_row is not None and _snap_row.price_rub is not None:
+ try:
+ with db.begin_nested():
+ upsert_listing_snapshot(
+ db,
+ listing_id=listing_id,
+ price_rub=int(_snap_row.price_rub),
+ price_per_m2=int(_snap_row.price_per_m2) if _snap_row.price_per_m2 else None,
+ run_id=None,
+ status="active",
+ )
+ _snap_written = True
+ except Exception as exc:
+ logger.warning(
+ "save_detail_enrichment: snapshot write failed listing_id=%s: %s",
+ listing_id,
+ exc,
+ )
+
+ # Price changes — INSERT new rows, de-dup by listing_id + change_time
+ for change in enrichment.price_changes:
+ if not change.get("change_time") or not change.get("price_rub"):
+ continue
+ db.execute(
+ text("""
+ INSERT INTO offer_price_history
+ (listing_id, change_time, price_rub, diff_percent, source)
+ VALUES (
+ CAST(:lid AS bigint),
+ CAST(:ct AS timestamptz),
+ CAST(:price AS numeric),
+ CAST(:diff AS numeric),
+ 'cian'
+ )
+ ON CONFLICT ON CONSTRAINT offer_price_history_listing_change_uq DO NOTHING
+ """),
+ {
+ "lid": listing_id,
+ "ct": change["change_time"],
+ "price": change["price_rub"],
+ "diff": change.get("diff_percent"),
+ },
+ )
+
+ # Agent profile UPSERT — store agent row and link to listing
+ if enrichment.agent_profile and enrichment.agent_profile.get("ext_agent_id"):
+ ap = enrichment.agent_profile
+ row = db.execute(
+ text("""
+ INSERT INTO agents
+ (ext_source, ext_agent_id, company_name, is_pro, skills, account_type)
+ VALUES (
+ 'cian',
+ :eai,
+ :cn,
+ :ip,
+ CAST(:sk AS jsonb),
+ :at
+ )
+ ON CONFLICT (ext_source, ext_agent_id) DO UPDATE SET
+ company_name = COALESCE(EXCLUDED.company_name, agents.company_name),
+ is_pro = COALESCE(EXCLUDED.is_pro, agents.is_pro),
+ skills = COALESCE(EXCLUDED.skills, agents.skills),
+ account_type = COALESCE(EXCLUDED.account_type, agents.account_type)
+ RETURNING id
+ """),
+ {
+ "eai": str(ap["ext_agent_id"]),
+ "cn": ap.get("company_name"),
+ "ip": ap.get("is_pro"),
+ "sk": json.dumps(ap.get("skills") or []),
+ "at": ap.get("account_type"),
+ },
+ )
+ agent_db_id = row.scalar_one_or_none()
+ if agent_db_id is not None:
+ db.execute(
+ text("""
+ UPDATE listings
+ SET agent_id_fk = CAST(:aid AS bigint)
+ WHERE id = CAST(:lid AS bigint)
+ AND agent_id_fk IS NULL
+ """),
+ {"aid": agent_db_id, "lid": listing_id},
+ )
+
+ db.commit()
+ logger.info(
+ "Cian detail enrichment saved for listing_id=%s (price_changes=%d, agent=%s, snapshot=%s)",
+ listing_id,
+ len(enrichment.price_changes),
+ bool(enrichment.agent_profile),
+ _snap_written,
+ )
diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/newbuilding.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/newbuilding.py
new file mode 100644
index 00000000..eb74c544
--- /dev/null
+++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/newbuilding.py
@@ -0,0 +1,792 @@
+"""Cian.ru newbuilding (ЖК) catalog page scraper.
+
+URL: https://zhk---i.cian.ru/
+MFE: 'newbuilding-card-desktop-frontend', key: 'initialState'
+
+Sister state containers extracted from same MFE initialState top-level keys:
+- realtyValuation: 7-month price chart (data.priceDynamics.chart.data.{labels,values})
+ → houses_price_dynamics
+- reliabilityState.reliability: наш.дом.рф checkStatus + details[] → house_reliability_checks
+- offers: grouped by roomType (fromDeveloperRooms[].layouts[])
+- reviews: house reviews list
+- housesByTurn: корпуса по очередям (stored as jsonb on houses)
+
+Stage 6 of CianScraper v1.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+from curl_cffi.requests import AsyncSession # type: ignore[import-untyped]
+
+from scraper_kit.browser_fetcher import BrowserFetcher
+from scraper_kit.cian_state_parser import extract_all_states, extract_state
+
+if TYPE_CHECKING:
+ from scraper_kit.contracts import ScraperConfig
+
+logger = logging.getLogger(__name__)
+
+# Canonical ЖК-slug subdomain URL embedded in any Cian SERP/page that references a
+# newbuilding (e.g. https://zhk-parkovyy-kvartal-ekb-i.cian.ru). The cat.php SERP for a
+# single newobject[] id puts this slug in its title block — extracting it is the WORKING
+# nb_id → ЖК-url resolution (#972; see resolve_cian_zhk_url_via_search).
+_ZHK_SLUG_URL_RE = re.compile(r"https://zhk-[a-z0-9-]+\.cian\.ru")
+
+# Title-anchored extraction (#972 hardening). The cat.php newbuilding-SERP renders the
+# canonical ЖК-slug of the QUERIED newobject as the href inside its page