diff --git a/tradein-mvp/backend/app/services/scrapers/cian_detail.py b/tradein-mvp/backend/app/services/scrapers/cian_detail.py index e7b4749c..6279c816 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian_detail.py +++ b/tradein-mvp/backend/app/services/scrapers/cian_detail.py @@ -44,8 +44,8 @@ class DetailEnrichment: windows_view_type: str | None = None # 'yard' / 'street' / 'yardAndStreet' separate_wcs_count: int | None = None combined_wcs_count: int | None = None - ceiling_height: float | None = None # meters - repair_type: str | None = None # 'cosmetic' / 'design' / 'no' — raw Cian value + ceiling_height: float | None = None # meters — offer.building.ceilingHeight (#1791) + repair_type: str | None = None # raw Cian: offer.repairType OR offer.decoration (#1791) repair_state: str | None = None # enum: needs_repair/standard/good/excellent kitchen_area_m2: float | None = None # offer.kitchenArea (м²) description: str | None = None # offer.description (текст объявления) @@ -141,7 +141,7 @@ async def fetch_detail( windows_view_type=_extract_windows_view(offer), separate_wcs_count=offer.get("separateWcsCount"), combined_wcs_count=offer.get("combinedWcsCount"), - ceiling_height=_parse_float(offer.get("ceilingHeight")), + ceiling_height=_extract_ceiling_height(offer), repair_type=_extract_repair_type(offer), repair_state=_extract_repair_state(offer), kitchen_area_m2=_parse_float(offer.get("kitchenArea")), @@ -190,18 +190,31 @@ def _extract_windows_view(offer: dict[str, Any]) -> str | None: return offer.get("windowsViewType") +def _extract_ceiling_height(offer: dict[str, Any]) -> float | None: + """Высота потолков (м). + + На detail-странице (defaultState) значение лежит в ``offer.building.ceilingHeight``, + а НЕ в ``offer.ceilingHeight`` (последнее всегда null) — #1791. Top-level оставлен + как fallback на случай иной структуры карточки. + """ + building = offer.get("building") or {} + return _parse_float(building.get("ceilingHeight") or offer.get("ceilingHeight")) + + def _extract_repair_type(offer: dict[str, Any]) -> str | None: # Cian: 'cosmetic' / 'design' / 'no' / 'euro' / ... - return offer.get("repairType") + # На detail-странице структурное поле чаще приходит как `decoration` + # ('preFine'/'fine'/...), а `repairType` пустое (#1791) — читаем оба. + return offer.get("repairType") or offer.get("decoration") def _extract_repair_state(offer: dict[str, Any]) -> str | None: - """Нормализованный enum из offer.repairType → needs_repair/standard/good/excellent. + """Нормализованный enum из repairType/decoration → needs_repair/standard/good/excellent. - Если структурное поле repairType пустое — fallback на инференс из описания (#622). + Если структурное поле пустое — fallback на инференс из описания (#622). Структурное значение всегда в приоритете. """ - state = normalize_repair_state(offer.get("repairType")) + state = normalize_repair_state(offer.get("repairType") or offer.get("decoration")) if state is None: state = infer_repair_state_from_text(offer.get("description")) return state diff --git a/tradein-mvp/backend/app/services/scrapers/repair_state_normalizer.py b/tradein-mvp/backend/app/services/scrapers/repair_state_normalizer.py index 8693347f..d31d70b2 100644 --- a/tradein-mvp/backend/app/services/scrapers/repair_state_normalizer.py +++ b/tradein-mvp/backend/app/services/scrapers/repair_state_normalizer.py @@ -14,6 +14,7 @@ Связь с estimator._REPAIR_COEF: needs_repair=0.94 / standard=1.00 / good=1.05 / excellent=1.10 """ + from __future__ import annotations import logging @@ -24,21 +25,19 @@ logger = logging.getLogger(__name__) # Все известные raw-значения с источником в комментарии _RAW_TO_ENUM: dict[str, str] = { # ── needs_repair ──────────────────────────────────────────────────────────── - "required": "needs_repair", # Avito: «требуется» / «без ремонта» (через _REPAIR_MAP) - "without": "needs_repair", # Cian: без ремонта - "no": "needs_repair", # Cian: нет ремонта (синоним without) - "rough": "needs_repair", # Cian/общий: черновая отделка - + "required": "needs_repair", # Avito: «требуется» / «без ремонта» (через _REPAIR_MAP) + "without": "needs_repair", # Cian: без ремонта + "no": "needs_repair", # Cian: нет ремонта (синоним without) + "rough": "needs_repair", # Cian/общий: черновая отделка + "prefine": "needs_repair", # Cian detail: предчистовая/whitebox (offer.decoration, #1791) # ── standard ──────────────────────────────────────────────────────────────── - "cosmetic": "standard", # Avito/Cian: косметический - + "cosmetic": "standard", # Avito/Cian: косметический # ── good ──────────────────────────────────────────────────────────────────── - "euro": "good", # Avito/Cian: евро - "fine": "good", # Cian: хорошая отделка (предчистовая / whitebox) - + "euro": "good", # Avito/Cian: евро + "fine": "good", # Cian: хорошая отделка (предчистовая / whitebox) # ── excellent ─────────────────────────────────────────────────────────────── - "designer": "excellent", # Avito: дизайнерский - "design": "excellent", # Cian: дизайнерский (camelCase вариант) + "designer": "excellent", # Avito: дизайнерский + "design": "excellent", # Cian: дизайнерский (camelCase вариант) } # Допустимые значения целевого enum — для pass-through защиты @@ -63,7 +62,9 @@ def normalize_repair_state(raw: str | None) -> str | None: # Pass-through: уже нормализованный enum (idempotency — нужно при ре-обработке) if raw in _VALID_ENUM: return raw - result = _RAW_TO_ENUM.get(raw) + # Cian отдаёт camelCase ('preFine', 'whiteBox') — ключи карты в нижнем регистре, + # сравниваем case-insensitive, чтобы не плодить дубль-ключи на каждый регистр. + result = _RAW_TO_ENUM.get(raw) or _RAW_TO_ENUM.get(raw.lower()) if result is None: logger.warning("repair_state_normalizer: unknown raw value %r — stored as NULL", raw) return result diff --git a/tradein-mvp/backend/tests/fixtures/cian_flat_330982715.html b/tradein-mvp/backend/tests/fixtures/cian_flat_330982715.html new file mode 100644 index 00000000..9110f591 --- /dev/null +++ b/tradein-mvp/backend/tests/fixtures/cian_flat_330982715.html @@ -0,0 +1,3 @@ + diff --git a/tradein-mvp/backend/tests/test_cian_detail.py b/tradein-mvp/backend/tests/test_cian_detail.py index d084da48..1eea78eb 100644 --- a/tradein-mvp/backend/tests/test_cian_detail.py +++ b/tradein-mvp/backend/tests/test_cian_detail.py @@ -2,6 +2,7 @@ from __future__ import annotations +from pathlib import Path from unittest.mock import AsyncMock, MagicMock import pytest @@ -9,13 +10,17 @@ import pytest from app.services.scrapers.cian_detail import ( DetailEnrichment, _extract_agent_profile, + _extract_ceiling_height, _extract_price_changes, _extract_repair_state, + _extract_repair_type, _parse_views, fetch_detail, save_detail_enrichment, ) +_FIXTURE_DIR = Path(__file__).parent / "fixtures" + # ── _parse_views ────────────────────────────────────────────────────────────── @@ -66,6 +71,41 @@ def test_repair_state_none_when_both_missing(): assert _extract_repair_state(offer) is None +# ── _extract_ceiling_height — offer.building.ceilingHeight, не offer.ceilingHeight (#1791) ── + + +def test_ceiling_height_from_building_node(): + """На detail-странице высота лежит в offer.building.ceilingHeight.""" + offer = {"ceilingHeight": None, "building": {"ceilingHeight": "3.0"}} + assert _extract_ceiling_height(offer) == 3.0 + + +def test_ceiling_height_falls_back_to_top_level(): + """Если building.ceilingHeight отсутствует — fallback на offer.ceilingHeight.""" + offer = {"ceilingHeight": "2.75", "building": {}} + assert _extract_ceiling_height(offer) == 2.75 + + +def test_ceiling_height_none_when_absent_everywhere(): + assert _extract_ceiling_height({"building": {}}) is None + assert _extract_ceiling_height({}) is None + + +# ── _extract_repair_type / state — offer.decoration fallback (#1791) ────────── + + +def test_repair_type_prefers_repair_type_then_decoration(): + assert _extract_repair_type({"repairType": "euro", "decoration": "cosmetic"}) == "euro" + assert _extract_repair_type({"repairType": None, "decoration": "preFine"}) == "preFine" + assert _extract_repair_type({}) is None + + +def test_repair_state_from_decoration_when_repair_type_missing(): + """offer.decoration='preFine' → needs_repair (через нормализатор).""" + offer = {"repairType": None, "decoration": "preFine", "description": ""} + assert _extract_repair_state(offer) == "needs_repair" + + # ── _extract_price_changes ──────────────────────────────────────────────────── @@ -690,3 +730,36 @@ async def test_fetch_detail_browser_fetcher_state_missing_returns_none(monkeypat ) assert result is None + + +# ── Real-card fixture (#1791) — снята с www.cian.ru/sale/flat/330982715/ ────── +# Регрессия на field-drift: ceiling/kitchen/description должны извлекаться из +# актуальной structure detail-страницы (defaultState, concat-формат). + + +@pytest.mark.asyncio +async def test_fetch_detail_real_fixture_extracts_drifted_fields(): + """На реальной карточке ceiling берётся из offer.building.ceilingHeight, + kitchen/description — из offer top-level. До #1791 ceiling был 0%.""" + html = (_FIXTURE_DIR / "cian_flat_330982715.html").read_text(encoding="utf-8") + + response = MagicMock() + response.status_code = 200 + response.text = html + + session = MagicMock() + session.get = AsyncMock(return_value=response) + session.close = AsyncMock() + + result = await fetch_detail("https://www.cian.ru/sale/flat/330982715/", session=session) + + assert result is not None + # ceiling_height — РАНЬШЕ был None (читали offer.ceilingHeight=null); + # теперь из offer.building.ceilingHeight=3.0 + assert result.ceiling_height == 3.0 + # kitchen_area_m2 — offer.kitchenArea='14.5' (строка → float) + assert result.kitchen_area_m2 == 14.5 + # description — непустой текст объявления + assert result.description is not None + assert len(result.description) > 50 + assert result.cian_id is not None diff --git a/tradein-mvp/backend/tests/test_repair_state_normalizer.py b/tradein-mvp/backend/tests/test_repair_state_normalizer.py index 67363ad3..e134926e 100644 --- a/tradein-mvp/backend/tests/test_repair_state_normalizer.py +++ b/tradein-mvp/backend/tests/test_repair_state_normalizer.py @@ -18,11 +18,16 @@ from app.services.scrapers.repair_state_normalizer import ( # ── needs_repair ───────────────────────────────────────────────────────────── -@pytest.mark.parametrize("raw", ["required", "without", "no", "rough"]) +@pytest.mark.parametrize("raw", ["required", "without", "no", "rough", "prefine"]) def test_needs_repair_values(raw: str) -> None: assert normalize_repair_state(raw) == "needs_repair" +def test_cian_camelcase_decoration_is_case_insensitive() -> None: + """Cian detail отдаёт offer.decoration='preFine' (camelCase) — #1791.""" + assert normalize_repair_state("preFine") == "needs_repair" + + # ── standard ─────────────────────────────────────────────────────────────────