From 440bb2ca51cbe781737eca7fc908eca7057bbef6 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 27 Jun 2026 17:45:11 +0000 Subject: [PATCH] feat(tradein/yandex): normalize house_type + promote kitchen/ceiling to columns (#2007) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit house_type normalized via shared house_type_normalizer (canon = _IMV_HOUSE_TYPE_MAP) — fixes estimator soft-penalty false-mismatch on ~70% yandex analogs; backfill migration 140 (SCREAMING->canon + 'other'->NULL). kitchen/ceiling promoted to columns (dashboard/matching, not valuation — see #2012); ceiling capped 2.0-6.0m to avoid numeric(3,2) overflow. Refs #2007 --- .../backend/app/services/scrapers/base.py | 28 +++++ .../scrapers/house_type_normalizer.py | 90 +++++++++++++++ .../app/services/scrapers/yandex_realty.py | 24 +++- .../sql/140_yandex_house_type_backfill.sql | 46 ++++++++ .../backend/tests/test_base_save_listings.py | 49 ++++++++ .../tests/test_house_type_normalizer.py | 108 ++++++++++++++++++ .../backend/tests/test_yandex_realty_serp.py | 18 ++- 7 files changed, 358 insertions(+), 5 deletions(-) create mode 100644 tradein-mvp/backend/app/services/scrapers/house_type_normalizer.py create mode 100644 tradein-mvp/backend/data/sql/140_yandex_house_type_backfill.sql create mode 100644 tradein-mvp/backend/tests/test_house_type_normalizer.py diff --git a/tradein-mvp/backend/app/services/scrapers/base.py b/tradein-mvp/backend/app/services/scrapers/base.py index 4cfd5893..2cb8b664 100644 --- a/tradein-mvp/backend/app/services/scrapers/base.py +++ b/tradein-mvp/backend/app/services/scrapers/base.py @@ -88,6 +88,12 @@ class ScrapedLot(BaseModel): has_balcony: bool | None = None kadastr_num: str | None = None + # Detail-поля кухни/потолка (#2007). Обычно из detail-страницы (avito_detail, + # cian_detail, yandex_detail), но yandex SERP отдаёт их прямо в карточке — + # промоутим из raw_payload в колонки для дашборда покрытия и matching. + kitchen_area_m2: float | None = None + ceiling_height_m: float | None = None + # Avito house linking (Stage 2a — search → houses) house_source: str | None = None # 'avito' / 'cian' house_ext_id: str | None = None # Avito's '3171365' @@ -339,6 +345,8 @@ def save_listings( "house_type": lot.house_type, "repair_state": lot.repair_state, "has_balcony": lot.has_balcony, + "kitchen_area_m2": lot.kitchen_area_m2, + "ceiling_height_m": lot.ceiling_height_m, "house_source": lot.house_source, "house_ext_id": lot.house_ext_id, "house_url": lot.house_url, @@ -385,6 +393,7 @@ def save_listings( address, lat, lon, region_code, rooms, area_m2, floor, total_floors, year_built, house_type, repair_state, has_balcony, + kitchen_area_m2, ceiling_height, ceiling_height_m, house_source, house_ext_id, house_url, listing_segment, newbuilding_id, newbuilding_url, price_rub, price_per_m2, @@ -404,6 +413,10 @@ def save_listings( :address, :lat, :lon, 66, :rooms, :area_m2, :floor, :total_floors, :year_built, :house_type, :repair_state, :has_balcony, + -- ceiling: один param :ceiling_height_m пишем в ОБЕ колонки — + -- ceiling_height (019, читает coverage-дашборд + yandex_detail/cian_detail) + -- и ceiling_height_m (111, живая avito-колонка). См. #2007. + :kitchen_area_m2, :ceiling_height_m, :ceiling_height_m, :house_source, :house_ext_id, :house_url, :listing_segment, :newbuilding_id, :newbuilding_url, :price_rub, :ppm2, @@ -449,6 +462,18 @@ def save_listings( metro_stations = EXCLUDED.metro_stations, listing_date = COALESCE(EXCLUDED.listing_date, listings.listing_date), area_m2 = COALESCE(EXCLUDED.area_m2, listings.area_m2), + -- kitchen/ceiling (#2007): COALESCE — SERP re-scrape источника без + -- этих полей (avito SERP → NULL) НЕ затирает detail-enriched значение + -- (avito_detail пишет ceiling_height_m отдельным UPDATE). + kitchen_area_m2 = COALESCE( + EXCLUDED.kitchen_area_m2, listings.kitchen_area_m2 + ), + ceiling_height = COALESCE( + EXCLUDED.ceiling_height, listings.ceiling_height + ), + ceiling_height_m = COALESCE( + EXCLUDED.ceiling_height_m, listings.ceiling_height_m + ), -- Yandex rich fields (+ shared description/agency/publish_date): -- COALESCE so a source that does not provide them never wipes -- a value previously written by another source / scrape. @@ -533,6 +558,9 @@ def save_listings( metro_stations = CAST(:metro_stations AS jsonb), listing_date = COALESCE(:listing_date, listing_date), area_m2 = COALESCE(:area_m2, area_m2), + kitchen_area_m2 = COALESCE(:kitchen_area_m2, kitchen_area_m2), + ceiling_height = COALESCE(:ceiling_height_m, ceiling_height), + ceiling_height_m = COALESCE(:ceiling_height_m, ceiling_height_m), publish_date = COALESCE(:publish_date, publish_date), days_on_market = COALESCE(:days_on_market, days_on_market), description = COALESCE(:description, description), diff --git a/tradein-mvp/backend/app/services/scrapers/house_type_normalizer.py b/tradein-mvp/backend/app/services/scrapers/house_type_normalizer.py new file mode 100644 index 00000000..f9ad280d --- /dev/null +++ b/tradein-mvp/backend/app/services/scrapers/house_type_normalizer.py @@ -0,0 +1,90 @@ +"""Нормализация house_type → каноничный enum для listings.house_type. + +Целевой канон (сверено с estimator._IMV_HOUSE_TYPE_MAP, estimator.py:146): + panel / brick / monolith / monolith_brick / block / wood. + +Зачем (#2007): estimator применяет soft-penalty по house_type — аналог с +house_type != target штрафуется. Yandex SERP отдаёт SCREAMING-значения +(MONOLIT / BRICK / PANEL / ...), которые НИКОГДА не равны каноничным +(monolith / brick / panel / ...) → ~70% yandex-аналогов получали ложный штраф +и фактически выпадали из скоринга. Нормализация на ингесте чинит это. + +Важно про None: неизвестное / 'other' / '' → None, НЕ 'other'. В estimator +`house_type IS NULL` нейтрально (без штрафа), а любое не-канон значение всегда +!= target → ложный штраф. Поэтому unknown лучше схлопнуть в NULL. + +Источники raw-значений: +- yandex SERP: building.buildingType — SCREAMING_SNAKE (MONOLIT, MONOLIT_BRICK, ...) +- cian SERP: building.materialType — camelCase (monolith, monolithBrick, + gasSilicateBlock, ...) — переиспользуется в #2008 (cian.py:815). +""" + +from __future__ import annotations + +import logging + +logger = logging.getLogger(__name__) + +# Канон listings.house_type — выровнен по estimator._IMV_HOUSE_TYPE_MAP. +_CANON: frozenset[str] = frozenset( + {"panel", "brick", "monolith", "monolith_brick", "block", "wood"} +) + +# raw-токен → канон. Ключи — точные значения вокабуляров источников; сравнение +# регистронезависимое (см. _LOOKUP ниже), но строго по полному токену енума, +# без fuzzy-матчинга. Неизвестные токены сюда НЕ попадают → normalize вернёт None. +_RAW_TO_CANON: dict[str, str] = { + # ── yandex SERP (SCREAMING_SNAKE buildingType) ─────────────────────────── + "MONOLIT": "monolith", + "BRICK": "brick", + "PANEL": "panel", + "MONOLIT_BRICK": "monolith_brick", + "BLOCK": "block", + "WOOD": "wood", + # ── cian SERP (camelCase materialType) — reuse в #2008 ─────────────────── + "monolith": "monolith", + "brick": "brick", + "panel": "panel", + "block": "block", + "wood": "wood", + "monolithBrick": "monolith_brick", + "gasSilicateBlock": "block", + "aerocreteBlock": "block", + "foamConcreteBlock": "block", + "stalin": "brick", # «сталинка» — кирпич +} + +# Регистронезависимый lookup. Лоуэркейс-ключи не коллизят между вокабулярами: +# 'MONOLIT'→'monolit' и 'monolith'→'monolith' — разные ключи. +_LOOKUP: dict[str, str] = {k.lower(): v for k, v in _RAW_TO_CANON.items()} + + +def normalize_house_type(raw: str | None) -> str | None: + """Преобразовать raw house_type в каноничный enum. + + Уже-каноничное значение возвращается as-is (идемпотентность — нужна при + ре-обработке и для backfill-миграции). Неизвестное / 'other' / пустое / + None → None (НЕ 'other': см. docstring модуля про estimator soft-penalty). + + Args: + raw: сырое значение из парсера (e.g. «MONOLIT», «monolithBrick») или None. + + Returns: + Одно из panel / brick / monolith / monolith_brick / block / wood, либо None. + """ + if raw is None: + return None + stripped = raw.strip() + if not stripped: + return None + # Pass-through: уже каноничное значение (idempotency — в т.ч. 'monolith_brick', + # которого нет среди raw-ключей карты). + if stripped in _CANON: + return stripped + # Регистронезависимый exact-token lookup по обоим вокабулярам. + canon = _LOOKUP.get(stripped) or _LOOKUP.get(stripped.lower()) + if canon is None: + # Неизвестное / 'other' — нормально (NULL нейтрально для estimator). + # debug, не warning: 'other' встречается массово, warning засорил бы лог. + logger.debug("house_type_normalizer: unmapped raw value %r — stored as NULL", raw) + return canon diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py index c8e5463e..a2344ff9 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py @@ -41,6 +41,7 @@ from curl_cffi.requests import AsyncSession as _CurlCffiSession from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper, ScrapedLot from app.services.scrapers.browser_fetcher import BrowserFetcher +from app.services.scrapers.house_type_normalizer import normalize_house_type from app.services.scrapers.price_brackets import get_price_seed_brackets logger = logging.getLogger(__name__) @@ -240,13 +241,13 @@ def _parse_gate_json( price.valuePerPart -> price_per_m2 area.value -> area_m2 livingSpace.value -> living_area_m2 - kitchenSpace.value -> raw_payload["kitchen_area_m2"] + kitchenSpace.value -> kitchen_area_m2 (+ raw_payload["kitchen_area_m2"]) roomsTotal (int | None) -> rooms (None -> 0 for studio) floorsOffered[0] -> floor floorsTotal -> total_floors building.builtYear -> year_built - building.buildingType -> house_type (raw: "MONOLIT"/"BRICK"/etc.) - ceilingHeight -> raw_payload["ceiling_height"] + building.buildingType -> house_type (normalize_house_type: MONOLIT->monolith) + ceilingHeight -> ceiling_height_m (+ raw_payload["ceiling_height"]) location.point.latitude/longitude -> lat / lon location.geocoderAddress -> address (full, with house number) mainImages[] -> photo_urls (prepend "https:", up to 5) @@ -307,6 +308,7 @@ def _entity_to_lot( kitchen_dict = entity.get("kitchenSpace") or {} kitchen_area_raw = kitchen_dict.get("value") + kitchen_area_m2 = float(kitchen_area_raw) if kitchen_area_raw is not None else None rooms_raw = entity.get("roomsTotal") if rooms_raw is None: @@ -323,11 +325,22 @@ def _entity_to_lot( total_floors = int(floors_total_raw) if floors_total_raw is not None else None ceiling_height = entity.get("ceilingHeight") + # Колонка ceiling_height — numeric(3,2), max 9.99. Yandex SERP отдаёт мусор + # (видели 18 м) → запись out-of-range уронила бы весь батч DataError'ом + # (per-lot SAVEPOINT ловит только IntegrityError). Берём только правдоподобный + # диапазон 2.0–6.0 м, иначе None; сырое значение остаётся в raw_payload. (#2007) + _ceiling_raw = float(ceiling_height) if ceiling_height is not None else None + ceiling_height_m = ( + _ceiling_raw if _ceiling_raw is not None and 2.0 <= _ceiling_raw <= 6.0 else None + ) building = entity.get("building") or {} year_built_raw = building.get("builtYear") year_built = int(year_built_raw) if year_built_raw is not None else None - house_type = building.get("buildingType") + # buildingType — SCREAMING (MONOLIT/BRICK/...). Нормализуем в канон, иначе + # estimator soft-penalty штрафует yandex-аналоги (#2007). + raw_building_type = building.get("buildingType") + house_type = normalize_house_type(raw_building_type) site_id = building.get("siteId") site_name = building.get("siteName") @@ -381,6 +394,8 @@ def _entity_to_lot( total_floors=total_floors, year_built=year_built, house_type=house_type, + kitchen_area_m2=kitchen_area_m2, + ceiling_height_m=ceiling_height_m, price_rub=price_rub, price_per_m2=price_per_m2, photo_urls=photo_urls, @@ -406,6 +421,7 @@ def _entity_to_lot( "page_param": page_param, "ceiling_height": ceiling_height, "kitchen_area_m2": kitchen_area_raw, + "raw_building_type": raw_building_type, # сырой SCREAMING — для отладки "site_name": site_name, "offer_id": offer_id, }, diff --git a/tradein-mvp/backend/data/sql/140_yandex_house_type_backfill.sql b/tradein-mvp/backend/data/sql/140_yandex_house_type_backfill.sql new file mode 100644 index 00000000..9f1168b7 --- /dev/null +++ b/tradein-mvp/backend/data/sql/140_yandex_house_type_backfill.sql @@ -0,0 +1,46 @@ +-- 140_yandex_house_type_backfill.sql +-- Purpose: нормализовать УЖЕ записанные yandex listings.house_type из SCREAMING +-- вокабуляра (MONOLIT/BRICK/PANEL/...) в канон (monolith/brick/panel/...). +-- +-- Rationale (#2007): +-- Yandex SERP писал building.buildingType как есть — SCREAMING_SNAKE. Estimator +-- применяет soft-penalty по house_type: аналог с house_type != target штрафуется. +-- 'MONOLIT' никогда не равен каноничному 'monolith' → ~70% yandex-аналогов +-- получали ложный штраф. Код-фикс (normalize_house_type на ингесте) чинит НОВЫЕ +-- строки; этот файл подтягивает СТАРЫЕ под тот же канон. +-- +-- Маппинг тождественен house_type_normalizer._RAW_TO_CANON (yandex-ветка): +-- MONOLIT->monolith, BRICK->brick, PANEL->panel, MONOLIT_BRICK->monolith_brick, +-- BLOCK->block, WOOD->wood. +-- +-- Плюс 'other' (не-канон легаси-значение, 299 строк) → NULL: паритет с +-- normalize_house_type (unknown→None). В estimator soft-penalty NULL нейтрально, +-- а 'other' != target всегда даёт ложный штраф. Лоуэркейс 'brick'/'panel' уже +-- каноничны — не трогаем. +-- +-- Idempotency: после первого прогона SCREAMING/'other'-значений не остаётся, +-- WHERE-фильтры пусты → повторный apply = no-op (0 rows). Scope строго +-- source='yandex' — cian/avito не трогаем (канон в коде / отдельных PR, напр. #2008). + +BEGIN; + +UPDATE listings + SET house_type = CASE house_type + WHEN 'MONOLIT' THEN 'monolith' + WHEN 'BRICK' THEN 'brick' + WHEN 'PANEL' THEN 'panel' + WHEN 'MONOLIT_BRICK' THEN 'monolith_brick' + WHEN 'BLOCK' THEN 'block' + WHEN 'WOOD' THEN 'wood' + ELSE house_type + END + WHERE source = 'yandex' + AND house_type IN ('MONOLIT', 'BRICK', 'PANEL', 'MONOLIT_BRICK', 'BLOCK', 'WOOD'); + +-- 'other' → NULL (паритет с normalize_house_type: unknown→None, нейтрально в estimator). +UPDATE listings + SET house_type = NULL + WHERE source = 'yandex' + AND house_type = 'other'; + +COMMIT; diff --git a/tradein-mvp/backend/tests/test_base_save_listings.py b/tradein-mvp/backend/tests/test_base_save_listings.py index 5ef70dde..31e88835 100644 --- a/tradein-mvp/backend/tests/test_base_save_listings.py +++ b/tradein-mvp/backend/tests/test_base_save_listings.py @@ -250,6 +250,55 @@ def test_save_listings_on_conflict_updates_cian_cols(): assert params["description_minhash"] == "newhash" +# ── Test 3b: kitchen/ceiling promote (#2007) ───────────────────────────────── + + +def test_save_listings_promotes_kitchen_ceiling_to_columns(): + """yandex lot с kitchen_area_m2/ceiling_height_m → params + обе ceiling-колонки. + + ceiling пишется ОДНИМ param :ceiling_height_m в ДВЕ колонки: ceiling_height + (019, coverage-дашборд + yandex_detail) и ceiling_height_m (111, avito-колонка). + """ + db = _mock_db(inserted=True) + lot = _base_lot( + source="yandex", + source_id=None, + source_url="https://realty.yandex.ru/offer/123/", + kitchen_area_m2=12.7, + ceiling_height_m=2.7, + ) + + save_listings(db, [lot]) + + params = _get_execute_params(db) + assert params["kitchen_area_m2"] == 12.7 + assert params["ceiling_height_m"] == 2.7 + + sql_text = _get_listings_insert_sql(db) + # обе колонки в списке INSERT + assert "kitchen_area_m2" in sql_text + assert "ceiling_height" in sql_text + assert "ceiling_height_m" in sql_text + # один param :ceiling_height_m питает обе ceiling-колонки (дважды в VALUES) + assert sql_text.count(":ceiling_height_m") >= 2 + # ON CONFLICT — COALESCE, чтобы re-scrape без полей не затирал detail-enrichment + assert "kitchen_area_m2 = COALESCE(" in sql_text + assert "ceiling_height = COALESCE(" in sql_text + assert "ceiling_height_m = COALESCE(" in sql_text + + +def test_save_listings_kitchen_ceiling_none_for_serp_without_them(): + """Avito SERP lot без kitchen/ceiling → params None (COALESCE сохранит detail-значение).""" + db = _mock_db(inserted=True) + lot = _base_lot(source="avito", source_id="987654") + + save_listings(db, [lot]) + + params = _get_execute_params(db) + assert params["kitchen_area_m2"] is None + assert params["ceiling_height_m"] is None + + # ── Test 4: empty list → early return, no db.execute call ─────────────────── diff --git a/tradein-mvp/backend/tests/test_house_type_normalizer.py b/tradein-mvp/backend/tests/test_house_type_normalizer.py new file mode 100644 index 00000000..cb26f914 --- /dev/null +++ b/tradein-mvp/backend/tests/test_house_type_normalizer.py @@ -0,0 +1,108 @@ +"""Unit-тесты для house_type_normalizer (#2007). + +Покрывают: +- normalize_house_type(): yandex SCREAMING + cian camelCase вокабуляры → канон, +- идемпотентность (normalize(normalize(x)) == normalize(x)), +- None / '' / whitespace / 'other' / unknown → None. +""" + +import pytest + +from app.services.scrapers.house_type_normalizer import ( + _CANON, + normalize_house_type, +) + +# ── yandex SERP (SCREAMING_SNAKE buildingType) ─────────────────────────────── + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("MONOLIT", "monolith"), + ("BRICK", "brick"), + ("PANEL", "panel"), + ("MONOLIT_BRICK", "monolith_brick"), + ("BLOCK", "block"), + ("WOOD", "wood"), + ], +) +def test_yandex_screaming_vocab(raw: str, expected: str) -> None: + assert normalize_house_type(raw) == expected + + +# ── cian SERP (camelCase materialType) — reuse #2008 ───────────────────────── + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("monolith", "monolith"), + ("brick", "brick"), + ("panel", "panel"), + ("block", "block"), + ("wood", "wood"), + ("monolithBrick", "monolith_brick"), + ("gasSilicateBlock", "block"), + ("aerocreteBlock", "block"), + ("foamConcreteBlock", "block"), + ("stalin", "brick"), + ], +) +def test_cian_camelcase_vocab(raw: str, expected: str) -> None: + assert normalize_house_type(raw) == expected + + +# ── регистронезависимость ──────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("monolit", "monolith"), # lower yandex-token + ("Brick", "brick"), + ("MONOLITHBRICK", "monolith_brick"), + (" PANEL ", "panel"), # окружающие пробелы + ], +) +def test_case_insensitive_and_strip(raw: str, expected: str) -> None: + assert normalize_house_type(raw) == expected + + +# ── идемпотентность ────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize("canon", sorted(_CANON)) +def test_canonical_passthrough(canon: str) -> None: + """Уже каноничное значение возвращается as-is (в т.ч. monolith_brick).""" + assert normalize_house_type(canon) == canon + + +@pytest.mark.parametrize( + "raw", + [ + "MONOLIT", + "MONOLIT_BRICK", + "monolithBrick", + "gasSilicateBlock", + "stalin", + "monolith_brick", + "panel", + "other", + None, + "", + ], +) +def test_idempotency(raw: str | None) -> None: + """normalize(normalize(x)) == normalize(x) для любого входа.""" + once = normalize_house_type(raw) + assert normalize_house_type(once) == once + + +# ── None / unknown → None ──────────────────────────────────────────────────── + + +@pytest.mark.parametrize("raw", [None, "", " ", "other", "OTHER", "unknown", " keramzit "]) +def test_unknown_returns_none(raw: str | None) -> None: + """Неизвестное / 'other' / пустое → None (НЕ 'other'): NULL нейтрально для estimator.""" + assert normalize_house_type(raw) is None diff --git a/tradein-mvp/backend/tests/test_yandex_realty_serp.py b/tradein-mvp/backend/tests/test_yandex_realty_serp.py index a74d6084..87d8a37c 100644 --- a/tradein-mvp/backend/tests/test_yandex_realty_serp.py +++ b/tradein-mvp/backend/tests/test_yandex_realty_serp.py @@ -222,7 +222,9 @@ def test_entity_to_lot_full_entity(): assert lot.floor == 8 assert lot.total_floors == 25 assert lot.year_built == 2016 - assert lot.house_type == "MONOLIT" + assert lot.house_type == "monolith" # normalize_house_type(MONOLIT) — #2007 + assert lot.kitchen_area_m2 == pytest.approx(12.7) # promoted to column — #2007 + assert lot.ceiling_height_m == pytest.approx(2.7) # promoted to column — #2007 assert lot.lat == pytest.approx(56.79512) assert lot.lon == pytest.approx(60.49186) assert lot.address == "Ekaterinburg, ulitsa Evgenia Savkova, 8" @@ -234,6 +236,20 @@ def test_entity_to_lot_full_entity(): assert lot.raw_payload is not None assert lot.raw_payload["ceiling_height"] == pytest.approx(2.7) assert lot.raw_payload["kitchen_area_m2"] == pytest.approx(12.7) + assert lot.raw_payload["raw_building_type"] == "MONOLIT" # сырой SCREAMING сохранён + + +def test_entity_to_lot_ceiling_out_of_range_dropped(): + """ceiling_height — numeric(3,2) (max 9.99); неправдоподобный yandex SERP мусор + (видели 18 м) отсекаем в None, иначе DataError уронил бы весь батч. Сырое + значение остаётся в raw_payload. (#2007)""" + lot_hi = _entity_to_lot({**_ENTITY_FULL, "ceilingHeight": 18}) + assert lot_hi is not None + assert lot_hi.ceiling_height_m is None + assert lot_hi.raw_payload["ceiling_height"] == 18 # сырое сохранено + lot_lo = _entity_to_lot({**_ENTITY_FULL, "ceilingHeight": 1.6}) + assert lot_lo is not None + assert lot_lo.ceiling_height_m is None def test_entity_to_lot_segment_vtorichka_default():