feat(tradein/yandex): normalize house_type + promote kitchen/ceiling to columns (#2007)

house_type normalized via shared house_type_normalizer (canon = _IMV_HOUSE_TYPE_MAP) — fixes estimator soft-penalty false-mismatch on ~70% yandex analogs; backfill migration 140 (SCREAMING->canon + 'other'->NULL). kitchen/ceiling promoted to columns (dashboard/matching, not valuation — see #2012); ceiling capped 2.0-6.0m to avoid numeric(3,2) overflow. Refs #2007

tradein-mvp/backend/app/services/scrapers/base.py

@@ -88,6 +88,12 @@ class ScrapedLot(BaseModel):
     has_balcony: bool | None = None
     kadastr_num: str | None = None
 
+    # Detail-поля кухни/потолка (#2007). Обычно из detail-страницы (avito_detail,
+    # cian_detail, yandex_detail), но yandex SERP отдаёт их прямо в карточке —
+    # промоутим из raw_payload в колонки для дашборда покрытия и matching.
+    kitchen_area_m2: float | None = None
+    ceiling_height_m: float | None = None
+
     # Avito house linking (Stage 2a — search → houses)
     house_source: str | None = None  # 'avito' / 'cian'
     house_ext_id: str | None = None  # Avito's '3171365'
@@ -339,6 +345,8 @@ def save_listings(
             "house_type": lot.house_type,
             "repair_state": lot.repair_state,
             "has_balcony": lot.has_balcony,
+            "kitchen_area_m2": lot.kitchen_area_m2,
+            "ceiling_height_m": lot.ceiling_height_m,
             "house_source": lot.house_source,
             "house_ext_id": lot.house_ext_id,
             "house_url": lot.house_url,
@@ -385,6 +393,7 @@ def save_listings(
                 address, lat, lon, region_code,
                 rooms, area_m2, floor, total_floors, year_built,
                 house_type, repair_state, has_balcony,
+                kitchen_area_m2, ceiling_height, ceiling_height_m,
                 house_source, house_ext_id, house_url, listing_segment,
                 newbuilding_id, newbuilding_url,
                 price_rub, price_per_m2,
@@ -404,6 +413,10 @@ def save_listings(
                 :address, :lat, :lon, 66,
                 :rooms, :area_m2, :floor, :total_floors, :year_built,
                 :house_type, :repair_state, :has_balcony,
+                -- ceiling: один param :ceiling_height_m пишем в ОБЕ колонки —
+                -- ceiling_height (019, читает coverage-дашборд + yandex_detail/cian_detail)
+                -- и ceiling_height_m (111, живая avito-колонка). См. #2007.
+                :kitchen_area_m2, :ceiling_height_m, :ceiling_height_m,
                 :house_source, :house_ext_id, :house_url, :listing_segment,
                 :newbuilding_id, :newbuilding_url,
                 :price_rub, :ppm2,
@@ -449,6 +462,18 @@ def save_listings(
                     metro_stations = EXCLUDED.metro_stations,
                     listing_date = COALESCE(EXCLUDED.listing_date, listings.listing_date),
                     area_m2 = COALESCE(EXCLUDED.area_m2, listings.area_m2),
+                    -- kitchen/ceiling (#2007): COALESCE — SERP re-scrape источника без
+                    -- этих полей (avito SERP → NULL) НЕ затирает detail-enriched значение
+                    -- (avito_detail пишет ceiling_height_m отдельным UPDATE).
+                    kitchen_area_m2 = COALESCE(
+                        EXCLUDED.kitchen_area_m2, listings.kitchen_area_m2
+                    ),
+                    ceiling_height = COALESCE(
+                        EXCLUDED.ceiling_height, listings.ceiling_height
+                    ),
+                    ceiling_height_m = COALESCE(
+                        EXCLUDED.ceiling_height_m, listings.ceiling_height_m
+                    ),
                     -- Yandex rich fields (+ shared description/agency/publish_date):
                     -- COALESCE so a source that does not provide them never wipes
                     -- a value previously written by another source / scrape.
@@ -533,6 +558,9 @@ def save_listings(
                                metro_stations = CAST(:metro_stations AS jsonb),
                                listing_date = COALESCE(:listing_date, listing_date),
                                area_m2 = COALESCE(:area_m2, area_m2),
+                               kitchen_area_m2 = COALESCE(:kitchen_area_m2, kitchen_area_m2),
+                               ceiling_height = COALESCE(:ceiling_height_m, ceiling_height),
+                               ceiling_height_m = COALESCE(:ceiling_height_m, ceiling_height_m),
                                publish_date = COALESCE(:publish_date, publish_date),
                                days_on_market = COALESCE(:days_on_market, days_on_market),
                                description = COALESCE(:description, description),

tradein-mvp/backend/app/services/scrapers/house_type_normalizer.py

@@ -0,0 +1,90 @@
+"""Нормализация house_type → каноничный enum для listings.house_type.
+
+Целевой канон (сверено с estimator._IMV_HOUSE_TYPE_MAP, estimator.py:146):
+  panel / brick / monolith / monolith_brick / block / wood.
+
+Зачем (#2007): estimator применяет soft-penalty по house_type — аналог с
+house_type != target штрафуется. Yandex SERP отдаёт SCREAMING-значения
+(MONOLIT / BRICK / PANEL / ...), которые НИКОГДА не равны каноничным
+(monolith / brick / panel / ...) → ~70% yandex-аналогов получали ложный штраф
+и фактически выпадали из скоринга. Нормализация на ингесте чинит это.
+
+Важно про None: неизвестное / 'other' / '' → None, НЕ 'other'. В estimator
+`house_type IS NULL` нейтрально (без штрафа), а любое не-канон значение всегда
+!= target → ложный штраф. Поэтому unknown лучше схлопнуть в NULL.
+
+Источники raw-значений:
+- yandex SERP: building.buildingType — SCREAMING_SNAKE (MONOLIT, MONOLIT_BRICK, ...)
+- cian SERP: building.materialType — camelCase (monolith, monolithBrick,
+  gasSilicateBlock, ...) — переиспользуется в #2008 (cian.py:815).
+"""
+
+from __future__ import annotations
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+# Канон listings.house_type — выровнен по estimator._IMV_HOUSE_TYPE_MAP.
+_CANON: frozenset[str] = frozenset(
+    {"panel", "brick", "monolith", "monolith_brick", "block", "wood"}
+)
+
+# raw-токен → канон. Ключи — точные значения вокабуляров источников; сравнение
+# регистронезависимое (см. _LOOKUP ниже), но строго по полному токену енума,
+# без fuzzy-матчинга. Неизвестные токены сюда НЕ попадают → normalize вернёт None.
+_RAW_TO_CANON: dict[str, str] = {
+    # ── yandex SERP (SCREAMING_SNAKE buildingType) ───────────────────────────
+    "MONOLIT": "monolith",
+    "BRICK": "brick",
+    "PANEL": "panel",
+    "MONOLIT_BRICK": "monolith_brick",
+    "BLOCK": "block",
+    "WOOD": "wood",
+    # ── cian SERP (camelCase materialType) — reuse в #2008 ───────────────────
+    "monolith": "monolith",
+    "brick": "brick",
+    "panel": "panel",
+    "block": "block",
+    "wood": "wood",
+    "monolithBrick": "monolith_brick",
+    "gasSilicateBlock": "block",
+    "aerocreteBlock": "block",
+    "foamConcreteBlock": "block",
+    "stalin": "brick",  # «сталинка» — кирпич
+}
+
+# Регистронезависимый lookup. Лоуэркейс-ключи не коллизят между вокабулярами:
+# 'MONOLIT'→'monolit' и 'monolith'→'monolith' — разные ключи.
+_LOOKUP: dict[str, str] = {k.lower(): v for k, v in _RAW_TO_CANON.items()}
+
+
+def normalize_house_type(raw: str | None) -> str | None:
+    """Преобразовать raw house_type в каноничный enum.
+
+    Уже-каноничное значение возвращается as-is (идемпотентность — нужна при
+    ре-обработке и для backfill-миграции). Неизвестное / 'other' / пустое /
+    None → None (НЕ 'other': см. docstring модуля про estimator soft-penalty).
+
+    Args:
+        raw: сырое значение из парсера (e.g. «MONOLIT», «monolithBrick») или None.
+
+    Returns:
+        Одно из panel / brick / monolith / monolith_brick / block / wood, либо None.
+    """
+    if raw is None:
+        return None
+    stripped = raw.strip()
+    if not stripped:
+        return None
+    # Pass-through: уже каноничное значение (idempotency — в т.ч. 'monolith_brick',
+    # которого нет среди raw-ключей карты).
+    if stripped in _CANON:
+        return stripped
+    # Регистронезависимый exact-token lookup по обоим вокабулярам.
+    canon = _LOOKUP.get(stripped) or _LOOKUP.get(stripped.lower())
+    if canon is None:
+        # Неизвестное / 'other' — нормально (NULL нейтрально для estimator).
+        # debug, не warning: 'other' встречается массово, warning засорил бы лог.
+        logger.debug("house_type_normalizer: unmapped raw value %r — stored as NULL", raw)
+    return canon

tradein-mvp/backend/app/services/scrapers/yandex_realty.py

@@ -41,6 +41,7 @@ from curl_cffi.requests import AsyncSession as _CurlCffiSession
 from app.services.scraper_settings import get_scraper_delay
 from app.services.scrapers.base import BaseScraper, ScrapedLot
 from app.services.scrapers.browser_fetcher import BrowserFetcher
+from app.services.scrapers.house_type_normalizer import normalize_house_type
 from app.services.scrapers.price_brackets import get_price_seed_brackets
 
 logger = logging.getLogger(__name__)
@@ -240,13 +241,13 @@ def _parse_gate_json(
       price.valuePerPart              -> price_per_m2
       area.value                      -> area_m2
       livingSpace.value               -> living_area_m2
-      kitchenSpace.value              -> raw_payload["kitchen_area_m2"]
+      kitchenSpace.value              -> kitchen_area_m2 (+ raw_payload["kitchen_area_m2"])
       roomsTotal (int | None)         -> rooms (None -> 0 for studio)
       floorsOffered[0]                -> floor
       floorsTotal                     -> total_floors
       building.builtYear              -> year_built
-      building.buildingType           -> house_type (raw: "MONOLIT"/"BRICK"/etc.)
-      ceilingHeight                   -> raw_payload["ceiling_height"]
+      building.buildingType           -> house_type (normalize_house_type: MONOLIT->monolith)
+      ceilingHeight                   -> ceiling_height_m (+ raw_payload["ceiling_height"])
       location.point.latitude/longitude -> lat / lon
       location.geocoderAddress        -> address (full, with house number)
       mainImages[]                    -> photo_urls (prepend "https:", up to 5)
@@ -307,6 +308,7 @@ def _entity_to_lot(
 
         kitchen_dict = entity.get("kitchenSpace") or {}
         kitchen_area_raw = kitchen_dict.get("value")
+        kitchen_area_m2 = float(kitchen_area_raw) if kitchen_area_raw is not None else None
 
         rooms_raw = entity.get("roomsTotal")
         if rooms_raw is None:
@@ -323,11 +325,22 @@ def _entity_to_lot(
         total_floors = int(floors_total_raw) if floors_total_raw is not None else None
 
         ceiling_height = entity.get("ceilingHeight")
+        # Колонка ceiling_height — numeric(3,2), max 9.99. Yandex SERP отдаёт мусор
+        # (видели 18 м) → запись out-of-range уронила бы весь батч DataError'ом
+        # (per-lot SAVEPOINT ловит только IntegrityError). Берём только правдоподобный
+        # диапазон 2.0–6.0 м, иначе None; сырое значение остаётся в raw_payload. (#2007)
+        _ceiling_raw = float(ceiling_height) if ceiling_height is not None else None
+        ceiling_height_m = (
+            _ceiling_raw if _ceiling_raw is not None and 2.0 <= _ceiling_raw <= 6.0 else None
+        )
 
         building = entity.get("building") or {}
         year_built_raw = building.get("builtYear")
         year_built = int(year_built_raw) if year_built_raw is not None else None
-        house_type = building.get("buildingType")
+        # buildingType — SCREAMING (MONOLIT/BRICK/...). Нормализуем в канон, иначе
+        # estimator soft-penalty штрафует yandex-аналоги (#2007).
+        raw_building_type = building.get("buildingType")
+        house_type = normalize_house_type(raw_building_type)
 
         site_id = building.get("siteId")
         site_name = building.get("siteName")
@@ -381,6 +394,8 @@ def _entity_to_lot(
             total_floors=total_floors,
             year_built=year_built,
             house_type=house_type,
+            kitchen_area_m2=kitchen_area_m2,
+            ceiling_height_m=ceiling_height_m,
             price_rub=price_rub,
             price_per_m2=price_per_m2,
             photo_urls=photo_urls,
@@ -406,6 +421,7 @@ def _entity_to_lot(
                 "page_param": page_param,
                 "ceiling_height": ceiling_height,
                 "kitchen_area_m2": kitchen_area_raw,
+                "raw_building_type": raw_building_type,  # сырой SCREAMING — для отладки
                 "site_name": site_name,
                 "offer_id": offer_id,
             },

tradein-mvp/backend/data/sql/140_yandex_house_type_backfill.sql

@@ -0,0 +1,46 @@
+-- 140_yandex_house_type_backfill.sql
+-- Purpose: нормализовать УЖЕ записанные yandex listings.house_type из SCREAMING
+--          вокабуляра (MONOLIT/BRICK/PANEL/...) в канон (monolith/brick/panel/...).
+--
+-- Rationale (#2007):
+--   Yandex SERP писал building.buildingType как есть — SCREAMING_SNAKE. Estimator
+--   применяет soft-penalty по house_type: аналог с house_type != target штрафуется.
+--   'MONOLIT' никогда не равен каноничному 'monolith' → ~70% yandex-аналогов
+--   получали ложный штраф. Код-фикс (normalize_house_type на ингесте) чинит НОВЫЕ
+--   строки; этот файл подтягивает СТАРЫЕ под тот же канон.
+--
+-- Маппинг тождественен house_type_normalizer._RAW_TO_CANON (yandex-ветка):
+--   MONOLIT->monolith, BRICK->brick, PANEL->panel, MONOLIT_BRICK->monolith_brick,
+--   BLOCK->block, WOOD->wood.
+--
+-- Плюс 'other' (не-канон легаси-значение, 299 строк) → NULL: паритет с
+--   normalize_house_type (unknown→None). В estimator soft-penalty NULL нейтрально,
+--   а 'other' != target всегда даёт ложный штраф. Лоуэркейс 'brick'/'panel' уже
+--   каноничны — не трогаем.
+--
+-- Idempotency: после первого прогона SCREAMING/'other'-значений не остаётся,
+--   WHERE-фильтры пусты → повторный apply = no-op (0 rows). Scope строго
+--   source='yandex' — cian/avito не трогаем (канон в коде / отдельных PR, напр. #2008).
+
+BEGIN;
+
+UPDATE listings
+   SET house_type = CASE house_type
+                        WHEN 'MONOLIT'       THEN 'monolith'
+                        WHEN 'BRICK'         THEN 'brick'
+                        WHEN 'PANEL'         THEN 'panel'
+                        WHEN 'MONOLIT_BRICK' THEN 'monolith_brick'
+                        WHEN 'BLOCK'         THEN 'block'
+                        WHEN 'WOOD'          THEN 'wood'
+                        ELSE house_type
+                    END
+ WHERE source = 'yandex'
+   AND house_type IN ('MONOLIT', 'BRICK', 'PANEL', 'MONOLIT_BRICK', 'BLOCK', 'WOOD');
+
+-- 'other' → NULL (паритет с normalize_house_type: unknown→None, нейтрально в estimator).
+UPDATE listings
+   SET house_type = NULL
+ WHERE source = 'yandex'
+   AND house_type = 'other';
+
+COMMIT;

tradein-mvp/backend/tests/test_base_save_listings.py

@@ -250,6 +250,55 @@ def test_save_listings_on_conflict_updates_cian_cols():
     assert params["description_minhash"] == "newhash"
 
 
+# ── Test 3b: kitchen/ceiling promote (#2007) ─────────────────────────────────
+
+
+def test_save_listings_promotes_kitchen_ceiling_to_columns():
+    """yandex lot с kitchen_area_m2/ceiling_height_m → params + обе ceiling-колонки.
+
+    ceiling пишется ОДНИМ param :ceiling_height_m в ДВЕ колонки: ceiling_height
+    (019, coverage-дашборд + yandex_detail) и ceiling_height_m (111, avito-колонка).
+    """
+    db = _mock_db(inserted=True)
+    lot = _base_lot(
+        source="yandex",
+        source_id=None,
+        source_url="https://realty.yandex.ru/offer/123/",
+        kitchen_area_m2=12.7,
+        ceiling_height_m=2.7,
+    )
+
+    save_listings(db, [lot])
+
+    params = _get_execute_params(db)
+    assert params["kitchen_area_m2"] == 12.7
+    assert params["ceiling_height_m"] == 2.7
+
+    sql_text = _get_listings_insert_sql(db)
+    # обе колонки в списке INSERT
+    assert "kitchen_area_m2" in sql_text
+    assert "ceiling_height" in sql_text
+    assert "ceiling_height_m" in sql_text
+    # один param :ceiling_height_m питает обе ceiling-колонки (дважды в VALUES)
+    assert sql_text.count(":ceiling_height_m") >= 2
+    # ON CONFLICT — COALESCE, чтобы re-scrape без полей не затирал detail-enrichment
+    assert "kitchen_area_m2 = COALESCE(" in sql_text
+    assert "ceiling_height = COALESCE(" in sql_text
+    assert "ceiling_height_m = COALESCE(" in sql_text
+
+
+def test_save_listings_kitchen_ceiling_none_for_serp_without_them():
+    """Avito SERP lot без kitchen/ceiling → params None (COALESCE сохранит detail-значение)."""
+    db = _mock_db(inserted=True)
+    lot = _base_lot(source="avito", source_id="987654")
+
+    save_listings(db, [lot])
+
+    params = _get_execute_params(db)
+    assert params["kitchen_area_m2"] is None
+    assert params["ceiling_height_m"] is None
+
+
 # ── Test 4: empty list → early return, no db.execute call ───────────────────
 
 

tradein-mvp/backend/tests/test_house_type_normalizer.py

@@ -0,0 +1,108 @@
+"""Unit-тесты для house_type_normalizer (#2007).
+
+Покрывают:
+- normalize_house_type(): yandex SCREAMING + cian camelCase вокабуляры → канон,
+- идемпотентность (normalize(normalize(x)) == normalize(x)),
+- None / '' / whitespace / 'other' / unknown → None.
+"""
+
+import pytest
+
+from app.services.scrapers.house_type_normalizer import (
+    _CANON,
+    normalize_house_type,
+)
+
+# ── yandex SERP (SCREAMING_SNAKE buildingType) ───────────────────────────────
+
+
+@pytest.mark.parametrize(
+    ("raw", "expected"),
+    [
+        ("MONOLIT", "monolith"),
+        ("BRICK", "brick"),
+        ("PANEL", "panel"),
+        ("MONOLIT_BRICK", "monolith_brick"),
+        ("BLOCK", "block"),
+        ("WOOD", "wood"),
+    ],
+)
+def test_yandex_screaming_vocab(raw: str, expected: str) -> None:
+    assert normalize_house_type(raw) == expected
+
+
+# ── cian SERP (camelCase materialType) — reuse #2008 ─────────────────────────
+
+
+@pytest.mark.parametrize(
+    ("raw", "expected"),
+    [
+        ("monolith", "monolith"),
+        ("brick", "brick"),
+        ("panel", "panel"),
+        ("block", "block"),
+        ("wood", "wood"),
+        ("monolithBrick", "monolith_brick"),
+        ("gasSilicateBlock", "block"),
+        ("aerocreteBlock", "block"),
+        ("foamConcreteBlock", "block"),
+        ("stalin", "brick"),
+    ],
+)
+def test_cian_camelcase_vocab(raw: str, expected: str) -> None:
+    assert normalize_house_type(raw) == expected
+
+
+# ── регистронезависимость ────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize(
+    ("raw", "expected"),
+    [
+        ("monolit", "monolith"),  # lower yandex-token
+        ("Brick", "brick"),
+        ("MONOLITHBRICK", "monolith_brick"),
+        ("  PANEL  ", "panel"),  # окружающие пробелы
+    ],
+)
+def test_case_insensitive_and_strip(raw: str, expected: str) -> None:
+    assert normalize_house_type(raw) == expected
+
+
+# ── идемпотентность ──────────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize("canon", sorted(_CANON))
+def test_canonical_passthrough(canon: str) -> None:
+    """Уже каноничное значение возвращается as-is (в т.ч. monolith_brick)."""
+    assert normalize_house_type(canon) == canon
+
+
+@pytest.mark.parametrize(
+    "raw",
+    [
+        "MONOLIT",
+        "MONOLIT_BRICK",
+        "monolithBrick",
+        "gasSilicateBlock",
+        "stalin",
+        "monolith_brick",
+        "panel",
+        "other",
+        None,
+        "",
+    ],
+)
+def test_idempotency(raw: str | None) -> None:
+    """normalize(normalize(x)) == normalize(x) для любого входа."""
+    once = normalize_house_type(raw)
+    assert normalize_house_type(once) == once
+
+
+# ── None / unknown → None ────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize("raw", [None, "", "   ", "other", "OTHER", "unknown", " keramzit "])
+def test_unknown_returns_none(raw: str | None) -> None:
+    """Неизвестное / 'other' / пустое → None (НЕ 'other'): NULL нейтрально для estimator."""
+    assert normalize_house_type(raw) is None

tradein-mvp/backend/tests/test_yandex_realty_serp.py

@@ -222,7 +222,9 @@ def test_entity_to_lot_full_entity():
     assert lot.floor == 8
     assert lot.total_floors == 25
     assert lot.year_built == 2016
-    assert lot.house_type == "MONOLIT"
+    assert lot.house_type == "monolith"  # normalize_house_type(MONOLIT) — #2007
+    assert lot.kitchen_area_m2 == pytest.approx(12.7)  # promoted to column — #2007
+    assert lot.ceiling_height_m == pytest.approx(2.7)  # promoted to column — #2007
     assert lot.lat == pytest.approx(56.79512)
     assert lot.lon == pytest.approx(60.49186)
     assert lot.address == "Ekaterinburg, ulitsa Evgenia Savkova, 8"
@@ -234,6 +236,20 @@ def test_entity_to_lot_full_entity():
     assert lot.raw_payload is not None
     assert lot.raw_payload["ceiling_height"] == pytest.approx(2.7)
     assert lot.raw_payload["kitchen_area_m2"] == pytest.approx(12.7)
+    assert lot.raw_payload["raw_building_type"] == "MONOLIT"  # сырой SCREAMING сохранён
+
+
+def test_entity_to_lot_ceiling_out_of_range_dropped():
+    """ceiling_height — numeric(3,2) (max 9.99); неправдоподобный yandex SERP мусор
+    (видели 18 м) отсекаем в None, иначе DataError уронил бы весь батч. Сырое
+    значение остаётся в raw_payload. (#2007)"""
+    lot_hi = _entity_to_lot({**_ENTITY_FULL, "ceilingHeight": 18})
+    assert lot_hi is not None
+    assert lot_hi.ceiling_height_m is None
+    assert lot_hi.raw_payload["ceiling_height"] == 18  # сырое сохранено
+    lot_lo = _entity_to_lot({**_ENTITY_FULL, "ceilingHeight": 1.6})
+    assert lot_lo is not None
+    assert lot_lo.ceiling_height_m is None
 
 
 def test_entity_to_lot_segment_vtorichka_default():