From 54a5d13c48a8ff78ead1c22e70c6774b6de91a7f Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 27 Jun 2026 16:35:25 +0300 Subject: [PATCH] =?UTF-8?q?feat(tradein/domclick):=20enrich=20Layer=20A=20?= =?UTF-8?q?SERP=20parser=20=E2=80=94=20description,=20repair-inference,=20?= =?UTF-8?q?minhash,=20seller?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Capture extra bff-search fields in _map_item (verified live): - description → ScrapedLot.description + infer_repair_state_from_text → repair_state (DomClick search has no renovation-enum; text inference like cian #622) - sha1 description_minhash (cross-source dedup) - agency_name from seller (defensive str|dict) - raw_payload: is_sber_collateral, has_discount, discount_value, duplicates_offer_count, seller Additive only; existing Layer A behavior unchanged. Deep card detail (renovation-enum, wallType, full price-history, owners_count) is card-only = QRATOR-blocked on prod proxy → separate effort (needs dedicated residential IP). --- .../backend/app/services/scrapers/domclick.py | 66 +++++++++++ .../backend/tests/scrapers/test_domclick.py | 104 ++++++++++++++++++ 2 files changed, 170 insertions(+) diff --git a/tradein-mvp/backend/app/services/scrapers/domclick.py b/tradein-mvp/backend/app/services/scrapers/domclick.py index b6f3087b..4ed48e0b 100644 --- a/tradein-mvp/backend/app/services/scrapers/domclick.py +++ b/tradein-mvp/backend/app/services/scrapers/domclick.py @@ -22,6 +22,7 @@ QRATOR банит прямые datacenter-запросы, но пропуска from __future__ import annotations +import hashlib import json import logging from datetime import date @@ -31,6 +32,7 @@ from urllib.parse import urlencode from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper, ScrapedLot from app.services.scrapers.domclick_exceptions import DomClickBlockedError +from app.services.scrapers.repair_state_normalizer import infer_repair_state_from_text logger = logging.getLogger(__name__) @@ -171,6 +173,27 @@ def _parse_publish_date(iso: str | None) -> date | None: return None +def _extract_agency_name(seller: dict[str, Any]) -> str | None: + """Извлечь читаемое имя агентства/агента из seller-блока BFF offer. + + seller.company / seller.agent могут быть строкой ИЛИ dict (defensive — + реальная форма поля варьируется). Приоритет: company → agent. Для dict + пробуем типичные name-поля. Возвращает None если ничего читаемого нет. + """ + for key in ("company", "agent"): + val = seller.get(key) + if isinstance(val, str): + name = val.strip() + if name: + return name + elif isinstance(val, dict): + for name_key in ("name", "title", "fullName"): + nm = val.get(name_key) + if isinstance(nm, str) and nm.strip(): + return nm.strip() + return None + + # ── DomClickScraper ────────────────────────────────────────────────────────── @@ -536,6 +559,30 @@ class DomClickScraper(BaseScraper): if price_per_m2 is None and area_m2 and area_m2 > 0: price_per_m2 = int(price / area_m2) + # ── Описание + repair-инференс + minhash (Layer A enrichment) ───── + # DomClick search не отдаёт renovation-enum: repair_state выводим из + # текста описания (как cian #622 fallback — «евроремонт»/«без отделки»). + # description_minhash — sha1 от описания (mirror cian) для cross-source + # дедупа. + description_raw = item.get("description") + description: str | None = None + if isinstance(description_raw, str): + description = description_raw.strip() or None + + repair_state: str | None = None + description_minhash: str | None = None + if description: + repair_state = infer_repair_state_from_text(description) + description_minhash = hashlib.sha1( + description.lower().encode("utf-8", errors="replace") + ).hexdigest()[:32] + + # ── Продавец / агентство ────────────────────────────────────────── + seller = item.get("seller") + if not isinstance(seller, dict): + seller = {} + agency_name = _extract_agency_name(seller) + flat_complex = item.get("flatComplex") or {} raw_payload: dict[str, Any] = { "isRosreestrApproved": item.get("isRosreestrApproved"), @@ -551,6 +598,21 @@ class DomClickScraper(BaseScraper): "updatedDate": item.get("updatedDate"), } + # ── Доп. SERP-поля (Layer A enrichment) — мержим в raw_payload, ──── + # сбрасывая None-значения чтобы payload оставался компактным; ничего + # из существующих ключей не теряем. + seller_compact = { + k: seller.get(k) for k in ("agent", "company") if seller.get(k) is not None + } + _enrich: dict[str, Any] = { + "is_sber_collateral": item.get("isSberCollateral"), + "has_discount": item.get("hasDiscount"), + "discount_value": item.get("discountValue"), + "duplicates_offer_count": item.get("duplicatesOfferCount"), + "seller": seller_compact or None, + } + raw_payload.update({k: v for k, v in _enrich.items() if v is not None}) + publish_date = _parse_publish_date(item.get("publishedDate")) return ScrapedLot( @@ -569,6 +631,10 @@ class DomClickScraper(BaseScraper): price_per_m2=price_per_m2, listing_segment="vtorichka", publish_date=publish_date, + description=description, + repair_state=repair_state, + description_minhash=description_minhash, + agency_name=agency_name, raw_payload=raw_payload, ) except Exception: diff --git a/tradein-mvp/backend/tests/scrapers/test_domclick.py b/tradein-mvp/backend/tests/scrapers/test_domclick.py index 4653d796..dc613020 100644 --- a/tradein-mvp/backend/tests/scrapers/test_domclick.py +++ b/tradein-mvp/backend/tests/scrapers/test_domclick.py @@ -259,6 +259,110 @@ def test_map_item_price_per_m2_fallback() -> None: assert lot.price_per_m2 == int(5_200_000 / 53.5) +# ── _map_item Layer A enrichment (description / repair / minhash / seller) ───── + +_VALID_REPAIR_ENUM = {"needs_repair", "standard", "good", "excellent"} + + +def test_map_item_description_repair_minhash() -> None: + """Offer с описанием → description set, repair_state выведен, minhash len 32.""" + scraper = DomClickScraper.__new__(DomClickScraper) + scraper.parse_failures = 0 + item = { + **_REALISTIC_ITEM, + "description": "Продаётся светлая квартира с евроремонтом, рядом метро.", + } + lot = scraper._map_item(item) + assert lot is not None + assert lot.description == "Продаётся светлая квартира с евроремонтом, рядом метро." + # «евроремонт» → good (см. repair_state_normalizer._TEXT_PATTERNS) + assert lot.repair_state == "good" + assert lot.repair_state in _VALID_REPAIR_ENUM + assert lot.description_minhash is not None + assert len(lot.description_minhash) == 32 + + +def test_map_item_description_stripped() -> None: + """description триммится; пустой/whitespace → None.""" + scraper = DomClickScraper.__new__(DomClickScraper) + scraper.parse_failures = 0 + item = {**_REALISTIC_ITEM, "description": " Без отделки, требуется ремонт. "} + lot = scraper._map_item(item) + assert lot is not None + assert lot.description == "Без отделки, требуется ремонт." + # «без отделк» → needs_repair + assert lot.repair_state == "needs_repair" + + +def test_map_item_enrichment_raw_payload() -> None: + """Новые SERP-поля попадают в raw_payload (is_sber_collateral и т.д.).""" + scraper = DomClickScraper.__new__(DomClickScraper) + scraper.parse_failures = 0 + item = { + **_REALISTIC_ITEM, + "isSberCollateral": True, + "hasDiscount": True, + "discountValue": 2.3, + "duplicatesOfferCount": 3, + "seller": {"company": "Этажи", "agent": "Иван Петров"}, + } + lot = scraper._map_item(item) + assert lot is not None + assert lot.raw_payload is not None + assert lot.raw_payload["is_sber_collateral"] is True + assert lot.raw_payload["has_discount"] is True + assert lot.raw_payload["discount_value"] == 2.3 + assert lot.raw_payload["duplicates_offer_count"] == 3 + assert lot.raw_payload["seller"] == {"company": "Этажи", "agent": "Иван Петров"} + # agency_name извлекается из seller.company (приоритет company → agent) + assert lot.agency_name == "Этажи" + + +def test_map_item_agency_name_from_agent_dict() -> None: + """seller.company отсутствует, agent — dict с name → agency_name = name.""" + scraper = DomClickScraper.__new__(DomClickScraper) + scraper.parse_failures = 0 + item = {**_REALISTIC_ITEM, "seller": {"agent": {"name": "Мария С."}}} + lot = scraper._map_item(item) + assert lot is not None + assert lot.agency_name == "Мария С." + assert lot.raw_payload is not None + assert lot.raw_payload["seller"] == {"agent": {"name": "Мария С."}} + + +def test_map_item_no_description_no_crash() -> None: + """Offer без description: не падает, description/repair_state/minhash = None.""" + scraper = DomClickScraper.__new__(DomClickScraper) + scraper.parse_failures = 0 + # _REALISTIC_ITEM не содержит description/seller/enrichment-полей + lot = scraper._map_item(_REALISTIC_ITEM) + assert lot is not None + assert lot.description is None + assert lot.repair_state is None + assert lot.description_minhash is None + assert lot.agency_name is None + + +def test_map_item_enrichment_preserves_existing_raw_payload_and_drops_none() -> None: + """Существующие raw_payload-ключи не теряются; None-enrich-ключи отброшены.""" + scraper = DomClickScraper.__new__(DomClickScraper) + scraper.parse_failures = 0 + lot = scraper._map_item(_REALISTIC_ITEM) + assert lot is not None + assert lot.raw_payload is not None + # Существующие ключи на месте + assert lot.raw_payload["isRosreestrApproved"] is True + assert lot.raw_payload["flatComplex"]["name"] == "ЖК Тест" + assert lot.raw_payload["lastPriceHistoryState"]["direction"] == "DECREASED" + assert "updatedDate" in lot.raw_payload + # None-значные enrich-ключи отсутствуют (компактный payload) + assert "is_sber_collateral" not in lot.raw_payload + assert "has_discount" not in lot.raw_payload + assert "discount_value" not in lot.raw_payload + assert "duplicates_offer_count" not in lot.raw_payload + assert "seller" not in lot.raw_payload + + # ── _is_geo_ok — гео-гард ────────────────────────────────────────────────────