feat(tradein/cian): promote kitchen/mortgage/apartments + normalize house_type (#2008)

cian house_type normalized via shared house_type_normalizer (monolithBrick->monolith_brick etc.) — fixes estimator soft-penalty false-mismatch (~1208 rows); backfill migration 141 + new columns is_apartments/is_rosreestr_checked. kitchen/mortgage/apartments promoted (dashboard/matching, not valuation — #2012). Refs #2008

tradein-mvp/backend/app/services/scrapers/base.py

@@ -94,6 +94,13 @@ class ScrapedLot(BaseModel):
     kitchen_area_m2: float | None = None
     ceiling_height_m: float | None = None
 
+    # Class B promote-поля (#2008, cian SERP). Для дашборда покрытия / matching,
+    # НЕ для valuation — estimator их пока НЕ читает (follow-up #2012). cian отдаёт
+    # их прямо в карточке; раньше лежали только в raw_payload.
+    mortgage_available: bool | None = None  # продавец допускает ипотеку
+    is_apartments: bool | None = None  # апартаменты (НЕ квартира — иной правовой статус)
+    is_rosreestr_checked: bool | None = None  # объявление сверено с ЕГРН
+
     # Avito house linking (Stage 2a — search → houses)
     house_source: str | None = None  # 'avito' / 'cian'
     house_ext_id: str | None = None  # Avito's '3171365'
@@ -347,6 +354,9 @@ def save_listings(
             "has_balcony": lot.has_balcony,
             "kitchen_area_m2": lot.kitchen_area_m2,
             "ceiling_height_m": lot.ceiling_height_m,
+            "mortgage_available": lot.mortgage_available,
+            "is_apartments": lot.is_apartments,
+            "is_rosreestr_checked": lot.is_rosreestr_checked,
             "house_source": lot.house_source,
             "house_ext_id": lot.house_ext_id,
             "house_url": lot.house_url,
@@ -394,6 +404,7 @@ def save_listings(
                 rooms, area_m2, floor, total_floors, year_built,
                 house_type, repair_state, has_balcony,
                 kitchen_area_m2, ceiling_height, ceiling_height_m,
+                mortgage_available, is_apartments, is_rosreestr_checked,
                 house_source, house_ext_id, house_url, listing_segment,
                 newbuilding_id, newbuilding_url,
                 price_rub, price_per_m2,
@@ -417,6 +428,7 @@ def save_listings(
                 -- ceiling_height (019, читает coverage-дашборд + yandex_detail/cian_detail)
                 -- и ceiling_height_m (111, живая avito-колонка). См. #2007.
                 :kitchen_area_m2, :ceiling_height_m, :ceiling_height_m,
+                :mortgage_available, :is_apartments, :is_rosreestr_checked,
                 :house_source, :house_ext_id, :house_url, :listing_segment,
                 :newbuilding_id, :newbuilding_url,
                 :price_rub, :ppm2,
@@ -474,6 +486,18 @@ def save_listings(
                     ceiling_height_m = COALESCE(
                         EXCLUDED.ceiling_height_m, listings.ceiling_height_m
                     ),
+                    -- Class B promote (#2008, cian SERP): COALESCE — re-scrape
+                    -- источника без поля (avito/yandex SERP → NULL) НЕ затирает
+                    -- значение, ранее записанное cian-ом / detail-ом.
+                    mortgage_available = COALESCE(
+                        EXCLUDED.mortgage_available, listings.mortgage_available
+                    ),
+                    is_apartments = COALESCE(
+                        EXCLUDED.is_apartments, listings.is_apartments
+                    ),
+                    is_rosreestr_checked = COALESCE(
+                        EXCLUDED.is_rosreestr_checked, listings.is_rosreestr_checked
+                    ),
                     -- Yandex rich fields (+ shared description/agency/publish_date):
                     -- COALESCE so a source that does not provide them never wipes
                     -- a value previously written by another source / scrape.
@@ -561,6 +585,13 @@ def save_listings(
                                kitchen_area_m2 = COALESCE(:kitchen_area_m2, kitchen_area_m2),
                                ceiling_height = COALESCE(:ceiling_height_m, ceiling_height),
                                ceiling_height_m = COALESCE(:ceiling_height_m, ceiling_height_m),
+                               mortgage_available = COALESCE(
+                                   :mortgage_available, mortgage_available
+                               ),
+                               is_apartments = COALESCE(:is_apartments, is_apartments),
+                               is_rosreestr_checked = COALESCE(
+                                   :is_rosreestr_checked, is_rosreestr_checked
+                               ),
                                publish_date = COALESCE(:publish_date, publish_date),
                                days_on_market = COALESCE(:days_on_market, days_on_market),
                                description = COALESCE(:description, description),

tradein-mvp/backend/app/services/scrapers/cian.py

@@ -41,6 +41,7 @@ from app.services.scraper_settings import get_scraper_delay
 from app.services.scrapers.base import BaseScraper, ScrapedLot
 from app.services.scrapers.browser_fetcher import BrowserFetcher
 from app.services.scrapers.cian_state_parser import extract_state
+from app.services.scrapers.house_type_normalizer import normalize_house_type
 from app.services.scrapers.price_brackets import get_price_seed_brackets
 from app.services.scrapers.repair_state_normalizer import (
     infer_repair_state_from_text,
@@ -803,8 +804,15 @@ class CianScraper(BaseScraper):
                 except (TypeError, ValueError):
                     pass
 
-            # kitchen_area_m2 сохраняем в raw_payload (поле не в ScrapedLot Stage 2)
+            # kitchen_area: сырое (str) в raw_payload + промоутим float в ScrapedLot
+            # колонку kitchen_area_m2 (#2008, Class B — дашборд/matching, не valuation).
             kitchen_area_raw = offer.get("kitchenArea")
+            kitchen_area_m2: float | None = None
+            if kitchen_area_raw is not None:
+                try:
+                    kitchen_area_m2 = float(kitchen_area_raw)
+                except (TypeError, ValueError):
+                    pass
 
             floor: int | None = offer.get("floorNumber")
 
@@ -812,7 +820,12 @@ class CianScraper(BaseScraper):
             building: dict[str, Any] = offer.get("building") or {}
             total_floors: int | None = building.get("floorsCount")
             year_built: int | None = building.get("buildYear")
-            house_type: str | None = building.get("materialType")
+            # house_type: cian materialType (camelCase: monolithBrick / gasSilicateBlock
+            # / stalin / ...) НЕ равен каноничному enum → estimator soft-penalty ложно
+            # штрафовал аналог. Нормализуем на ингесте (#2008, Class A); сырьё — в
+            # raw_payload['raw_material_type'] для аудита.
+            raw_material_type: str | None = building.get("materialType")
+            house_type: str | None = normalize_house_type(raw_material_type)
 
             # Newbuilding deadline year — если нет buildYear
             if year_built is None:
@@ -929,6 +942,13 @@ class CianScraper(BaseScraper):
             sale_type: str | None = bargain.get("saleType")
             bargain_allowed: bool | None = bargain.get("bargainAllowed")
 
+            # ── Class B promote (#2008): ипотека / апартаменты / проверка ЕГРН ──
+            # Раньше лежали только в raw_payload; промоутим в колонки listings для
+            # дашборда покрытия и matching (НЕ для valuation — estimator их не читает).
+            mortgage_available: bool | None = bargain.get("mortgageAllowed")
+            is_apartments: bool | None = offer.get("isApartments")
+            is_rosreestr_checked: bool | None = offer.get("isRosreestrChecked")
+
             # ── Description + minhash ─────────────────────────────────────────
             description: str | None = offer.get("description")
             # Cian предоставляет готовый minhash для dedup и cross-source matching.
@@ -975,7 +995,10 @@ class CianScraper(BaseScraper):
                 "offer_type": offer.get("offerType"),
                 "category": offer.get("category"),
                 "has_furniture": has_furniture,
-                "kitchen_area_m2": kitchen_area_raw,  # не в ScrapedLot — сохраняем здесь
+                # сырой kitchenArea (str) — колонка kitchen_area_m2 (float) промоучена выше
+                "kitchen_area_m2": kitchen_area_raw,
+                # сырой materialType (camelCase) — house_type нормализован выше (#2008)
+                "raw_material_type": raw_material_type,
                 "mortgage_allowed": bargain.get("mortgageAllowed"),
                 "sale_type": sale_type,
                 "newbuilding_name": newbuilding.get("name"),
@@ -1009,6 +1032,10 @@ class CianScraper(BaseScraper):
                 house_type=house_type,
                 repair_state=repair_state,
                 has_balcony=has_balcony,
+                kitchen_area_m2=kitchen_area_m2,
+                mortgage_available=mortgage_available,
+                is_apartments=is_apartments,
+                is_rosreestr_checked=is_rosreestr_checked,
                 kadastr_num=cadastral_number,
                 house_source=house_source,
                 house_ext_id=house_ext_id,

tradein-mvp/backend/data/sql/141_cian_promote_house_type.sql

@@ -0,0 +1,49 @@
+-- 141_cian_promote_house_type.sql
+-- Purpose (#2008):
+--   (1) Class B promote-колонки is_apartments / is_rosreestr_checked — cian SERP
+--       отдаёт их в карточке; раньше копились только в raw_payload. Нужны для
+--       дашборда покрытия и matching (НЕ для valuation — estimator их не читает,
+--       follow-up #2012). mortgage_available и kitchen_area_m2 УЖЕ существуют
+--       (011_listings_alter.sql) — не добавляем.
+--   (2) Backfill cian listings.house_type из camelCase materialType-вокабуляра в
+--       канон (monolith/brick/panel/monolith_brick/block/wood). Cian писал
+--       materialType как есть (monolithBrick / gasSilicateBlock / stalin / ...),
+--       что != каноничному enum → estimator soft-penalty ложно штрафовал аналог.
+--       Код-фикс (normalize_house_type на ингесте) чинит НОВЫЕ строки; этот файл
+--       подтягивает СТАРЫЕ под тот же канон.
+--
+-- Маппинг тождественен house_type_normalizer._RAW_TO_CANON (cian-ветка):
+--   monolithBrick->monolith_brick, gasSilicateBlock->block, aerocreteBlock->block,
+--   foamConcreteBlock->block, stalin->brick («сталинка» = кирпич).
+--   Уже-каноничные monolith/brick/panel/block/wood НЕ трогаем (ELSE house_type) —
+--   консервативно, чтобы не потерять канон. Неизвестные токены тоже остаются
+--   как есть (паритет с normalize_house_type, который их вернул бы NULL только на
+--   ингесте; здесь СТАРЫЕ значения не схлопываем агрессивно).
+--
+-- Idempotency:
+--   - ADD COLUMN IF NOT EXISTS — повторный apply безопасен.
+--   - WHERE house_type IN (<только мапимые camelCase>) → после первого прогона их
+--     не остаётся, повторный apply = no-op (0 rows). Scope строго source='cian' —
+--     yandex (#2007 / миграция 140) и avito не трогаем.
+
+BEGIN;
+
+ALTER TABLE listings ADD COLUMN IF NOT EXISTS is_apartments boolean;
+ALTER TABLE listings ADD COLUMN IF NOT EXISTS is_rosreestr_checked boolean;
+
+UPDATE listings
+   SET house_type = CASE house_type
+                        WHEN 'monolithBrick'     THEN 'monolith_brick'
+                        WHEN 'gasSilicateBlock'  THEN 'block'
+                        WHEN 'aerocreteBlock'    THEN 'block'
+                        WHEN 'foamConcreteBlock' THEN 'block'
+                        WHEN 'stalin'            THEN 'brick'
+                        ELSE house_type
+                    END
+ WHERE source = 'cian'
+   AND house_type IN (
+       'monolithBrick', 'gasSilicateBlock', 'aerocreteBlock',
+       'foamConcreteBlock', 'stalin'
+   );
+
+COMMIT;

tradein-mvp/backend/tests/test_base_save_listings.py

@@ -299,6 +299,50 @@ def test_save_listings_kitchen_ceiling_none_for_serp_without_them():
     assert params["ceiling_height_m"] is None
 
 
+# ── Test 3c: Class B promote (#2008) — mortgage/apartments/rosreestr ──────────
+
+
+def test_save_listings_promotes_class_b_cian_fields():
+    """#2008: mortgage_available/is_apartments/is_rosreestr_checked → params + SQL.
+
+    Колонки + ON CONFLICT COALESCE (re-scrape источника без полей не затирает
+    значение, ранее записанное cian-ом).
+    """
+    db = _mock_db(inserted=True)
+    lot = _cian_lot(
+        mortgage_available=True,
+        is_apartments=False,
+        is_rosreestr_checked=True,
+    )
+
+    save_listings(db, [lot])
+
+    params = _get_execute_params(db)
+    assert params["mortgage_available"] is True
+    assert params["is_apartments"] is False
+    assert params["is_rosreestr_checked"] is True
+
+    sql_text = _get_listings_insert_sql(db)
+    for col in ("mortgage_available", "is_apartments", "is_rosreestr_checked"):
+        assert col in sql_text, f"Column '{col}' missing from INSERT SQL"
+    assert "mortgage_available = COALESCE(" in sql_text
+    assert "is_apartments = COALESCE(" in sql_text
+    assert "is_rosreestr_checked = COALESCE(" in sql_text
+
+
+def test_save_listings_class_b_fields_none_for_non_cian():
+    """#2008: avito lot без Class B-полей → params None (COALESCE сохранит prior)."""
+    db = _mock_db(inserted=True)
+    lot = _base_lot(source="avito", source_id="987654")
+
+    save_listings(db, [lot])
+
+    params = _get_execute_params(db)
+    assert params["mortgage_available"] is None
+    assert params["is_apartments"] is None
+    assert params["is_rosreestr_checked"] is None
+
+
 # ── Test 4: empty list → early return, no db.execute call ───────────────────
 
 

tradein-mvp/backend/tests/test_cian_serp_scraper.py

@@ -200,7 +200,8 @@ def test_cian_serp_coords_no_jitter():
 
 
 def test_cian_serp_kitchen_area_in_raw_payload():
-    """kitchen_area_m2 не в ScrapedLot, но сохраняется в raw_payload."""
+    """Сырой kitchenArea (str) остаётся в raw_payload (#2008 промоутит float
+    в колонку ScrapedLot.kitchen_area_m2 отдельно — см. соответств. тест)."""
     offer = _make_full_offer(kitchen_area="11.5")
     html = _make_cian_serp_html([offer])
 
@@ -400,6 +401,69 @@ def test_cian_serp_building_fields():
     assert lot.total_floors == 9
 
 
+def test_cian_serp_promotes_class_b_fields():
+    """#2008: kitchen_area_m2 / mortgage_available / is_apartments /
+    is_rosreestr_checked промоутятся в колонки ScrapedLot (Class B —
+    дашборд/matching, НЕ valuation)."""
+    # _make_full_offer: mortgageAllowed=True, isApartments=False, isRosreestrChecked=True
+    offer = _make_full_offer(kitchen_area="11.5")
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+
+    lot = lots[0]
+    assert lot.kitchen_area_m2 == pytest.approx(11.5)
+    assert lot.mortgage_available is True
+    assert lot.is_apartments is False
+    assert lot.is_rosreestr_checked is True
+
+
+def test_cian_serp_house_type_normalized_monolith_brick():
+    """#2008 (Class A): cian materialType 'monolithBrick' → канон 'monolith_brick';
+    сырьё остаётся в raw_payload['raw_material_type']."""
+    offer = _make_full_offer(material_type="monolithBrick")
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+
+    lot = lots[0]
+    assert lot.house_type == "monolith_brick"
+    assert lot.raw_payload is not None
+    assert lot.raw_payload["raw_material_type"] == "monolithBrick"
+
+
+def test_cian_serp_house_type_block_synonyms_normalized():
+    """#2008: gasSilicate/aerocrete/foamConcrete Block + stalin → канон."""
+    scraper = CianScraper()
+    for raw, canon in (
+        ("gasSilicateBlock", "block"),
+        ("aerocreteBlock", "block"),
+        ("foamConcreteBlock", "block"),
+        ("stalin", "brick"),  # «сталинка» = кирпич
+    ):
+        offer = _make_full_offer(material_type=raw)
+        lots = scraper._parse_serp_html(_make_cian_serp_html([offer]))
+        assert lots[0].house_type == canon, f"{raw} → {lots[0].house_type} != {canon}"
+
+
+def test_cian_serp_house_type_canonical_passthrough():
+    """#2008: уже-каноничные monolith/panel остаются неизменными (idempotency)."""
+    scraper = CianScraper()
+    for raw in ("monolith", "panel", "brick", "block", "wood"):
+        offer = _make_full_offer(material_type=raw)
+        lots = scraper._parse_serp_html(_make_cian_serp_html([offer]))
+        assert lots[0].house_type == raw
+
+
+def test_cian_serp_house_type_unknown_becomes_none():
+    """#2008: неизвестный materialType → None (нейтрально для estimator soft-penalty)."""
+    offer = _make_full_offer(material_type="someUnknownMaterial")
+    lots = CianScraper()._parse_serp_html(_make_cian_serp_html([offer]))
+    assert lots[0].house_type is None
+
+
 def test_cian_serp_living_area():
     """living_area_m2 из offer.livingArea."""
     offer = _make_full_offer(living_area="35.0")