From c5227a164e0deb38575e8728b5faea43dd24ac8f Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 27 Jun 2026 18:01:50 +0000 Subject: [PATCH] feat(tradein/cian): promote kitchen/mortgage/apartments + normalize house_type (#2008) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cian house_type normalized via shared house_type_normalizer (monolithBrick->monolith_brick etc.) — fixes estimator soft-penalty false-mismatch (~1208 rows); backfill migration 141 + new columns is_apartments/is_rosreestr_checked. kitchen/mortgage/apartments promoted (dashboard/matching, not valuation — #2012). Refs #2008 --- .../backend/app/services/scrapers/base.py | 31 +++++++++ .../backend/app/services/scrapers/cian.py | 33 +++++++++- .../data/sql/141_cian_promote_house_type.sql | 49 ++++++++++++++ .../backend/tests/test_base_save_listings.py | 44 +++++++++++++ .../backend/tests/test_cian_serp_scraper.py | 66 ++++++++++++++++++- 5 files changed, 219 insertions(+), 4 deletions(-) create mode 100644 tradein-mvp/backend/data/sql/141_cian_promote_house_type.sql diff --git a/tradein-mvp/backend/app/services/scrapers/base.py b/tradein-mvp/backend/app/services/scrapers/base.py index 2cb8b664..305125d2 100644 --- a/tradein-mvp/backend/app/services/scrapers/base.py +++ b/tradein-mvp/backend/app/services/scrapers/base.py @@ -94,6 +94,13 @@ class ScrapedLot(BaseModel): kitchen_area_m2: float | None = None ceiling_height_m: float | None = None + # Class B promote-поля (#2008, cian SERP). Для дашборда покрытия / matching, + # НЕ для valuation — estimator их пока НЕ читает (follow-up #2012). cian отдаёт + # их прямо в карточке; раньше лежали только в raw_payload. + mortgage_available: bool | None = None # продавец допускает ипотеку + is_apartments: bool | None = None # апартаменты (НЕ квартира — иной правовой статус) + is_rosreestr_checked: bool | None = None # объявление сверено с ЕГРН + # Avito house linking (Stage 2a — search → houses) house_source: str | None = None # 'avito' / 'cian' house_ext_id: str | None = None # Avito's '3171365' @@ -347,6 +354,9 @@ def save_listings( "has_balcony": lot.has_balcony, "kitchen_area_m2": lot.kitchen_area_m2, "ceiling_height_m": lot.ceiling_height_m, + "mortgage_available": lot.mortgage_available, + "is_apartments": lot.is_apartments, + "is_rosreestr_checked": lot.is_rosreestr_checked, "house_source": lot.house_source, "house_ext_id": lot.house_ext_id, "house_url": lot.house_url, @@ -394,6 +404,7 @@ def save_listings( rooms, area_m2, floor, total_floors, year_built, house_type, repair_state, has_balcony, kitchen_area_m2, ceiling_height, ceiling_height_m, + mortgage_available, is_apartments, is_rosreestr_checked, house_source, house_ext_id, house_url, listing_segment, newbuilding_id, newbuilding_url, price_rub, price_per_m2, @@ -417,6 +428,7 @@ def save_listings( -- ceiling_height (019, читает coverage-дашборд + yandex_detail/cian_detail) -- и ceiling_height_m (111, живая avito-колонка). См. #2007. :kitchen_area_m2, :ceiling_height_m, :ceiling_height_m, + :mortgage_available, :is_apartments, :is_rosreestr_checked, :house_source, :house_ext_id, :house_url, :listing_segment, :newbuilding_id, :newbuilding_url, :price_rub, :ppm2, @@ -474,6 +486,18 @@ def save_listings( ceiling_height_m = COALESCE( EXCLUDED.ceiling_height_m, listings.ceiling_height_m ), + -- Class B promote (#2008, cian SERP): COALESCE — re-scrape + -- источника без поля (avito/yandex SERP → NULL) НЕ затирает + -- значение, ранее записанное cian-ом / detail-ом. + mortgage_available = COALESCE( + EXCLUDED.mortgage_available, listings.mortgage_available + ), + is_apartments = COALESCE( + EXCLUDED.is_apartments, listings.is_apartments + ), + is_rosreestr_checked = COALESCE( + EXCLUDED.is_rosreestr_checked, listings.is_rosreestr_checked + ), -- Yandex rich fields (+ shared description/agency/publish_date): -- COALESCE so a source that does not provide them never wipes -- a value previously written by another source / scrape. @@ -561,6 +585,13 @@ def save_listings( kitchen_area_m2 = COALESCE(:kitchen_area_m2, kitchen_area_m2), ceiling_height = COALESCE(:ceiling_height_m, ceiling_height), ceiling_height_m = COALESCE(:ceiling_height_m, ceiling_height_m), + mortgage_available = COALESCE( + :mortgage_available, mortgage_available + ), + is_apartments = COALESCE(:is_apartments, is_apartments), + is_rosreestr_checked = COALESCE( + :is_rosreestr_checked, is_rosreestr_checked + ), publish_date = COALESCE(:publish_date, publish_date), days_on_market = COALESCE(:days_on_market, days_on_market), description = COALESCE(:description, description), diff --git a/tradein-mvp/backend/app/services/scrapers/cian.py b/tradein-mvp/backend/app/services/scrapers/cian.py index 335b4be8..2841379b 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian.py +++ b/tradein-mvp/backend/app/services/scrapers/cian.py @@ -41,6 +41,7 @@ from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper, ScrapedLot from app.services.scrapers.browser_fetcher import BrowserFetcher from app.services.scrapers.cian_state_parser import extract_state +from app.services.scrapers.house_type_normalizer import normalize_house_type from app.services.scrapers.price_brackets import get_price_seed_brackets from app.services.scrapers.repair_state_normalizer import ( infer_repair_state_from_text, @@ -803,8 +804,15 @@ class CianScraper(BaseScraper): except (TypeError, ValueError): pass - # kitchen_area_m2 сохраняем в raw_payload (поле не в ScrapedLot Stage 2) + # kitchen_area: сырое (str) в raw_payload + промоутим float в ScrapedLot + # колонку kitchen_area_m2 (#2008, Class B — дашборд/matching, не valuation). kitchen_area_raw = offer.get("kitchenArea") + kitchen_area_m2: float | None = None + if kitchen_area_raw is not None: + try: + kitchen_area_m2 = float(kitchen_area_raw) + except (TypeError, ValueError): + pass floor: int | None = offer.get("floorNumber") @@ -812,7 +820,12 @@ class CianScraper(BaseScraper): building: dict[str, Any] = offer.get("building") or {} total_floors: int | None = building.get("floorsCount") year_built: int | None = building.get("buildYear") - house_type: str | None = building.get("materialType") + # house_type: cian materialType (camelCase: monolithBrick / gasSilicateBlock + # / stalin / ...) НЕ равен каноничному enum → estimator soft-penalty ложно + # штрафовал аналог. Нормализуем на ингесте (#2008, Class A); сырьё — в + # raw_payload['raw_material_type'] для аудита. + raw_material_type: str | None = building.get("materialType") + house_type: str | None = normalize_house_type(raw_material_type) # Newbuilding deadline year — если нет buildYear if year_built is None: @@ -929,6 +942,13 @@ class CianScraper(BaseScraper): sale_type: str | None = bargain.get("saleType") bargain_allowed: bool | None = bargain.get("bargainAllowed") + # ── Class B promote (#2008): ипотека / апартаменты / проверка ЕГРН ── + # Раньше лежали только в raw_payload; промоутим в колонки listings для + # дашборда покрытия и matching (НЕ для valuation — estimator их не читает). + mortgage_available: bool | None = bargain.get("mortgageAllowed") + is_apartments: bool | None = offer.get("isApartments") + is_rosreestr_checked: bool | None = offer.get("isRosreestrChecked") + # ── Description + minhash ───────────────────────────────────────── description: str | None = offer.get("description") # Cian предоставляет готовый minhash для dedup и cross-source matching. @@ -975,7 +995,10 @@ class CianScraper(BaseScraper): "offer_type": offer.get("offerType"), "category": offer.get("category"), "has_furniture": has_furniture, - "kitchen_area_m2": kitchen_area_raw, # не в ScrapedLot — сохраняем здесь + # сырой kitchenArea (str) — колонка kitchen_area_m2 (float) промоучена выше + "kitchen_area_m2": kitchen_area_raw, + # сырой materialType (camelCase) — house_type нормализован выше (#2008) + "raw_material_type": raw_material_type, "mortgage_allowed": bargain.get("mortgageAllowed"), "sale_type": sale_type, "newbuilding_name": newbuilding.get("name"), @@ -1009,6 +1032,10 @@ class CianScraper(BaseScraper): house_type=house_type, repair_state=repair_state, has_balcony=has_balcony, + kitchen_area_m2=kitchen_area_m2, + mortgage_available=mortgage_available, + is_apartments=is_apartments, + is_rosreestr_checked=is_rosreestr_checked, kadastr_num=cadastral_number, house_source=house_source, house_ext_id=house_ext_id, diff --git a/tradein-mvp/backend/data/sql/141_cian_promote_house_type.sql b/tradein-mvp/backend/data/sql/141_cian_promote_house_type.sql new file mode 100644 index 00000000..41e8bb9d --- /dev/null +++ b/tradein-mvp/backend/data/sql/141_cian_promote_house_type.sql @@ -0,0 +1,49 @@ +-- 141_cian_promote_house_type.sql +-- Purpose (#2008): +-- (1) Class B promote-колонки is_apartments / is_rosreestr_checked — cian SERP +-- отдаёт их в карточке; раньше копились только в raw_payload. Нужны для +-- дашборда покрытия и matching (НЕ для valuation — estimator их не читает, +-- follow-up #2012). mortgage_available и kitchen_area_m2 УЖЕ существуют +-- (011_listings_alter.sql) — не добавляем. +-- (2) Backfill cian listings.house_type из camelCase materialType-вокабуляра в +-- канон (monolith/brick/panel/monolith_brick/block/wood). Cian писал +-- materialType как есть (monolithBrick / gasSilicateBlock / stalin / ...), +-- что != каноничному enum → estimator soft-penalty ложно штрафовал аналог. +-- Код-фикс (normalize_house_type на ингесте) чинит НОВЫЕ строки; этот файл +-- подтягивает СТАРЫЕ под тот же канон. +-- +-- Маппинг тождественен house_type_normalizer._RAW_TO_CANON (cian-ветка): +-- monolithBrick->monolith_brick, gasSilicateBlock->block, aerocreteBlock->block, +-- foamConcreteBlock->block, stalin->brick («сталинка» = кирпич). +-- Уже-каноничные monolith/brick/panel/block/wood НЕ трогаем (ELSE house_type) — +-- консервативно, чтобы не потерять канон. Неизвестные токены тоже остаются +-- как есть (паритет с normalize_house_type, который их вернул бы NULL только на +-- ингесте; здесь СТАРЫЕ значения не схлопываем агрессивно). +-- +-- Idempotency: +-- - ADD COLUMN IF NOT EXISTS — повторный apply безопасен. +-- - WHERE house_type IN (<только мапимые camelCase>) → после первого прогона их +-- не остаётся, повторный apply = no-op (0 rows). Scope строго source='cian' — +-- yandex (#2007 / миграция 140) и avito не трогаем. + +BEGIN; + +ALTER TABLE listings ADD COLUMN IF NOT EXISTS is_apartments boolean; +ALTER TABLE listings ADD COLUMN IF NOT EXISTS is_rosreestr_checked boolean; + +UPDATE listings + SET house_type = CASE house_type + WHEN 'monolithBrick' THEN 'monolith_brick' + WHEN 'gasSilicateBlock' THEN 'block' + WHEN 'aerocreteBlock' THEN 'block' + WHEN 'foamConcreteBlock' THEN 'block' + WHEN 'stalin' THEN 'brick' + ELSE house_type + END + WHERE source = 'cian' + AND house_type IN ( + 'monolithBrick', 'gasSilicateBlock', 'aerocreteBlock', + 'foamConcreteBlock', 'stalin' + ); + +COMMIT; diff --git a/tradein-mvp/backend/tests/test_base_save_listings.py b/tradein-mvp/backend/tests/test_base_save_listings.py index 31e88835..9fbbf148 100644 --- a/tradein-mvp/backend/tests/test_base_save_listings.py +++ b/tradein-mvp/backend/tests/test_base_save_listings.py @@ -299,6 +299,50 @@ def test_save_listings_kitchen_ceiling_none_for_serp_without_them(): assert params["ceiling_height_m"] is None +# ── Test 3c: Class B promote (#2008) — mortgage/apartments/rosreestr ────────── + + +def test_save_listings_promotes_class_b_cian_fields(): + """#2008: mortgage_available/is_apartments/is_rosreestr_checked → params + SQL. + + Колонки + ON CONFLICT COALESCE (re-scrape источника без полей не затирает + значение, ранее записанное cian-ом). + """ + db = _mock_db(inserted=True) + lot = _cian_lot( + mortgage_available=True, + is_apartments=False, + is_rosreestr_checked=True, + ) + + save_listings(db, [lot]) + + params = _get_execute_params(db) + assert params["mortgage_available"] is True + assert params["is_apartments"] is False + assert params["is_rosreestr_checked"] is True + + sql_text = _get_listings_insert_sql(db) + for col in ("mortgage_available", "is_apartments", "is_rosreestr_checked"): + assert col in sql_text, f"Column '{col}' missing from INSERT SQL" + assert "mortgage_available = COALESCE(" in sql_text + assert "is_apartments = COALESCE(" in sql_text + assert "is_rosreestr_checked = COALESCE(" in sql_text + + +def test_save_listings_class_b_fields_none_for_non_cian(): + """#2008: avito lot без Class B-полей → params None (COALESCE сохранит prior).""" + db = _mock_db(inserted=True) + lot = _base_lot(source="avito", source_id="987654") + + save_listings(db, [lot]) + + params = _get_execute_params(db) + assert params["mortgage_available"] is None + assert params["is_apartments"] is None + assert params["is_rosreestr_checked"] is None + + # ── Test 4: empty list → early return, no db.execute call ─────────────────── diff --git a/tradein-mvp/backend/tests/test_cian_serp_scraper.py b/tradein-mvp/backend/tests/test_cian_serp_scraper.py index 47dcb999..a7d3a49f 100644 --- a/tradein-mvp/backend/tests/test_cian_serp_scraper.py +++ b/tradein-mvp/backend/tests/test_cian_serp_scraper.py @@ -200,7 +200,8 @@ def test_cian_serp_coords_no_jitter(): def test_cian_serp_kitchen_area_in_raw_payload(): - """kitchen_area_m2 не в ScrapedLot, но сохраняется в raw_payload.""" + """Сырой kitchenArea (str) остаётся в raw_payload (#2008 промоутит float + в колонку ScrapedLot.kitchen_area_m2 отдельно — см. соответств. тест).""" offer = _make_full_offer(kitchen_area="11.5") html = _make_cian_serp_html([offer]) @@ -400,6 +401,69 @@ def test_cian_serp_building_fields(): assert lot.total_floors == 9 +def test_cian_serp_promotes_class_b_fields(): + """#2008: kitchen_area_m2 / mortgage_available / is_apartments / + is_rosreestr_checked промоутятся в колонки ScrapedLot (Class B — + дашборд/matching, НЕ valuation).""" + # _make_full_offer: mortgageAllowed=True, isApartments=False, isRosreestrChecked=True + offer = _make_full_offer(kitchen_area="11.5") + html = _make_cian_serp_html([offer]) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + + lot = lots[0] + assert lot.kitchen_area_m2 == pytest.approx(11.5) + assert lot.mortgage_available is True + assert lot.is_apartments is False + assert lot.is_rosreestr_checked is True + + +def test_cian_serp_house_type_normalized_monolith_brick(): + """#2008 (Class A): cian materialType 'monolithBrick' → канон 'monolith_brick'; + сырьё остаётся в raw_payload['raw_material_type'].""" + offer = _make_full_offer(material_type="monolithBrick") + html = _make_cian_serp_html([offer]) + + scraper = CianScraper() + lots = scraper._parse_serp_html(html) + + lot = lots[0] + assert lot.house_type == "monolith_brick" + assert lot.raw_payload is not None + assert lot.raw_payload["raw_material_type"] == "monolithBrick" + + +def test_cian_serp_house_type_block_synonyms_normalized(): + """#2008: gasSilicate/aerocrete/foamConcrete Block + stalin → канон.""" + scraper = CianScraper() + for raw, canon in ( + ("gasSilicateBlock", "block"), + ("aerocreteBlock", "block"), + ("foamConcreteBlock", "block"), + ("stalin", "brick"), # «сталинка» = кирпич + ): + offer = _make_full_offer(material_type=raw) + lots = scraper._parse_serp_html(_make_cian_serp_html([offer])) + assert lots[0].house_type == canon, f"{raw} → {lots[0].house_type} != {canon}" + + +def test_cian_serp_house_type_canonical_passthrough(): + """#2008: уже-каноничные monolith/panel остаются неизменными (idempotency).""" + scraper = CianScraper() + for raw in ("monolith", "panel", "brick", "block", "wood"): + offer = _make_full_offer(material_type=raw) + lots = scraper._parse_serp_html(_make_cian_serp_html([offer])) + assert lots[0].house_type == raw + + +def test_cian_serp_house_type_unknown_becomes_none(): + """#2008: неизвестный materialType → None (нейтрально для estimator soft-penalty).""" + offer = _make_full_offer(material_type="someUnknownMaterial") + lots = CianScraper()._parse_serp_html(_make_cian_serp_html([offer])) + assert lots[0].house_type is None + + def test_cian_serp_living_area(): """living_area_m2 из offer.livingArea.""" offer = _make_full_offer(living_area="35.0")