Merge pull request 'feat(tradein/domclick): обогащение Layer A поиска — description, repair-инференс, minhash, seller' (#1999) from feat/tradein-domclick-search-enrich into main

tradein-mvp/backend/app/services/scrapers/domclick.py

@@ -22,6 +22,7 @@ QRATOR банит прямые datacenter-запросы, но пропуска
 
 from __future__ import annotations
 
+import hashlib
 import json
 import logging
 from datetime import date
@@ -31,6 +32,7 @@ from urllib.parse import urlencode
 from app.services.scraper_settings import get_scraper_delay
 from app.services.scrapers.base import BaseScraper, ScrapedLot
 from app.services.scrapers.domclick_exceptions import DomClickBlockedError
+from app.services.scrapers.repair_state_normalizer import infer_repair_state_from_text
 
 logger = logging.getLogger(__name__)
 
@@ -171,6 +173,27 @@ def _parse_publish_date(iso: str | None) -> date | None:
         return None
 
 
+def _extract_agency_name(seller: dict[str, Any]) -> str | None:
+    """Извлечь читаемое имя агентства/агента из seller-блока BFF offer.
+
+    seller.company / seller.agent могут быть строкой ИЛИ dict (defensive —
+    реальная форма поля варьируется). Приоритет: company → agent. Для dict
+    пробуем типичные name-поля. Возвращает None если ничего читаемого нет.
+    """
+    for key in ("company", "agent"):
+        val = seller.get(key)
+        if isinstance(val, str):
+            name = val.strip()
+            if name:
+                return name
+        elif isinstance(val, dict):
+            for name_key in ("name", "title", "fullName"):
+                nm = val.get(name_key)
+                if isinstance(nm, str) and nm.strip():
+                    return nm.strip()
+    return None
+
+
 # ── DomClickScraper ──────────────────────────────────────────────────────────
 
 
@@ -536,6 +559,30 @@ class DomClickScraper(BaseScraper):
             if price_per_m2 is None and area_m2 and area_m2 > 0:
                 price_per_m2 = int(price / area_m2)
 
+            # ── Описание + repair-инференс + minhash (Layer A enrichment) ─────
+            # DomClick search не отдаёт renovation-enum: repair_state выводим из
+            # текста описания (как cian #622 fallback — «евроремонт»/«без отделки»).
+            # description_minhash — sha1 от описания (mirror cian) для cross-source
+            # дедупа.
+            description_raw = item.get("description")
+            description: str | None = None
+            if isinstance(description_raw, str):
+                description = description_raw.strip() or None
+
+            repair_state: str | None = None
+            description_minhash: str | None = None
+            if description:
+                repair_state = infer_repair_state_from_text(description)
+                description_minhash = hashlib.sha1(
+                    description.lower().encode("utf-8", errors="replace")
+                ).hexdigest()[:32]
+
+            # ── Продавец / агентство ──────────────────────────────────────────
+            seller = item.get("seller")
+            if not isinstance(seller, dict):
+                seller = {}
+            agency_name = _extract_agency_name(seller)
+
             flat_complex = item.get("flatComplex") or {}
             raw_payload: dict[str, Any] = {
                 "isRosreestrApproved": item.get("isRosreestrApproved"),
@@ -551,6 +598,21 @@ class DomClickScraper(BaseScraper):
                 "updatedDate": item.get("updatedDate"),
             }
 
+            # ── Доп. SERP-поля (Layer A enrichment) — мержим в raw_payload, ────
+            # сбрасывая None-значения чтобы payload оставался компактным; ничего
+            # из существующих ключей не теряем.
+            seller_compact = {
+                k: seller.get(k) for k in ("agent", "company") if seller.get(k) is not None
+            }
+            _enrich: dict[str, Any] = {
+                "is_sber_collateral": item.get("isSberCollateral"),
+                "has_discount": item.get("hasDiscount"),
+                "discount_value": item.get("discountValue"),
+                "duplicates_offer_count": item.get("duplicatesOfferCount"),
+                "seller": seller_compact or None,
+            }
+            raw_payload.update({k: v for k, v in _enrich.items() if v is not None})
+
             publish_date = _parse_publish_date(item.get("publishedDate"))
 
             return ScrapedLot(
@@ -569,6 +631,10 @@ class DomClickScraper(BaseScraper):
                 price_per_m2=price_per_m2,
                 listing_segment="vtorichka",
                 publish_date=publish_date,
+                description=description,
+                repair_state=repair_state,
+                description_minhash=description_minhash,
+                agency_name=agency_name,
                 raw_payload=raw_payload,
             )
         except Exception:

tradein-mvp/backend/tests/scrapers/test_domclick.py

@@ -259,6 +259,110 @@ def test_map_item_price_per_m2_fallback() -> None:
     assert lot.price_per_m2 == int(5_200_000 / 53.5)
 
 
+# ── _map_item Layer A enrichment (description / repair / minhash / seller) ─────
+
+_VALID_REPAIR_ENUM = {"needs_repair", "standard", "good", "excellent"}
+
+
+def test_map_item_description_repair_minhash() -> None:
+    """Offer с описанием → description set, repair_state выведен, minhash len 32."""
+    scraper = DomClickScraper.__new__(DomClickScraper)
+    scraper.parse_failures = 0
+    item = {
+        **_REALISTIC_ITEM,
+        "description": "Продаётся светлая квартира с евроремонтом, рядом метро.",
+    }
+    lot = scraper._map_item(item)
+    assert lot is not None
+    assert lot.description == "Продаётся светлая квартира с евроремонтом, рядом метро."
+    # «евроремонт» → good (см. repair_state_normalizer._TEXT_PATTERNS)
+    assert lot.repair_state == "good"
+    assert lot.repair_state in _VALID_REPAIR_ENUM
+    assert lot.description_minhash is not None
+    assert len(lot.description_minhash) == 32
+
+
+def test_map_item_description_stripped() -> None:
+    """description триммится; пустой/whitespace → None."""
+    scraper = DomClickScraper.__new__(DomClickScraper)
+    scraper.parse_failures = 0
+    item = {**_REALISTIC_ITEM, "description": "   Без отделки, требуется ремонт.  "}
+    lot = scraper._map_item(item)
+    assert lot is not None
+    assert lot.description == "Без отделки, требуется ремонт."
+    # «без отделк» → needs_repair
+    assert lot.repair_state == "needs_repair"
+
+
+def test_map_item_enrichment_raw_payload() -> None:
+    """Новые SERP-поля попадают в raw_payload (is_sber_collateral и т.д.)."""
+    scraper = DomClickScraper.__new__(DomClickScraper)
+    scraper.parse_failures = 0
+    item = {
+        **_REALISTIC_ITEM,
+        "isSberCollateral": True,
+        "hasDiscount": True,
+        "discountValue": 2.3,
+        "duplicatesOfferCount": 3,
+        "seller": {"company": "Этажи", "agent": "Иван Петров"},
+    }
+    lot = scraper._map_item(item)
+    assert lot is not None
+    assert lot.raw_payload is not None
+    assert lot.raw_payload["is_sber_collateral"] is True
+    assert lot.raw_payload["has_discount"] is True
+    assert lot.raw_payload["discount_value"] == 2.3
+    assert lot.raw_payload["duplicates_offer_count"] == 3
+    assert lot.raw_payload["seller"] == {"company": "Этажи", "agent": "Иван Петров"}
+    # agency_name извлекается из seller.company (приоритет company → agent)
+    assert lot.agency_name == "Этажи"
+
+
+def test_map_item_agency_name_from_agent_dict() -> None:
+    """seller.company отсутствует, agent — dict с name → agency_name = name."""
+    scraper = DomClickScraper.__new__(DomClickScraper)
+    scraper.parse_failures = 0
+    item = {**_REALISTIC_ITEM, "seller": {"agent": {"name": "Мария С."}}}
+    lot = scraper._map_item(item)
+    assert lot is not None
+    assert lot.agency_name == "Мария С."
+    assert lot.raw_payload is not None
+    assert lot.raw_payload["seller"] == {"agent": {"name": "Мария С."}}
+
+
+def test_map_item_no_description_no_crash() -> None:
+    """Offer без description: не падает, description/repair_state/minhash = None."""
+    scraper = DomClickScraper.__new__(DomClickScraper)
+    scraper.parse_failures = 0
+    # _REALISTIC_ITEM не содержит description/seller/enrichment-полей
+    lot = scraper._map_item(_REALISTIC_ITEM)
+    assert lot is not None
+    assert lot.description is None
+    assert lot.repair_state is None
+    assert lot.description_minhash is None
+    assert lot.agency_name is None
+
+
+def test_map_item_enrichment_preserves_existing_raw_payload_and_drops_none() -> None:
+    """Существующие raw_payload-ключи не теряются; None-enrich-ключи отброшены."""
+    scraper = DomClickScraper.__new__(DomClickScraper)
+    scraper.parse_failures = 0
+    lot = scraper._map_item(_REALISTIC_ITEM)
+    assert lot is not None
+    assert lot.raw_payload is not None
+    # Существующие ключи на месте
+    assert lot.raw_payload["isRosreestrApproved"] is True
+    assert lot.raw_payload["flatComplex"]["name"] == "ЖК Тест"
+    assert lot.raw_payload["lastPriceHistoryState"]["direction"] == "DECREASED"
+    assert "updatedDate" in lot.raw_payload
+    # None-значные enrich-ключи отсутствуют (компактный payload)
+    assert "is_sber_collateral" not in lot.raw_payload
+    assert "has_discount" not in lot.raw_payload
+    assert "discount_value" not in lot.raw_payload
+    assert "duplicates_offer_count" not in lot.raw_payload
+    assert "seller" not in lot.raw_payload
+
+
 # ── _is_geo_ok — гео-гард ────────────────────────────────────────────────────