From 08509b1b78d8f3117be6eb884dd40ab43e4c349c Mon Sep 17 00:00:00 2001
From: bot-backend <bot-backend@gendsgn.local>
Date: Sat, 30 May 2026 20:28:56 +0000
Subject: [PATCH] =?UTF-8?q?fix(scrapers):=20normalize=20cian=20description?=
 =?UTF-8?q?Minhash=20list[int]=E2=86=92str=20+=20silent-0=20guard=20(Refs?=
 =?UTF-8?q?=20#836)=20(#840)=20Co-authored-by:=20bot-backend=20<bot-backen?=
 =?UTF-8?q?d@gendsgn.local>=20Co-committed-by:=20bot-backend=20<bot-backen?=
 =?UTF-8?q?d@gendsgn.local>?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../backend/app/services/scrapers/cian.py     | 33 +++++++++++++++++--
 .../backend/tests/test_cian_serp_scraper.py   | 26 +++++++++++++--
 2 files changed, 55 insertions(+), 4 deletions(-)

diff --git a/tradein-mvp/backend/app/services/scrapers/cian.py b/tradein-mvp/backend/app/services/scrapers/cian.py
index 42d0adf0..505ea902 100644
--- a/tradein-mvp/backend/app/services/scrapers/cian.py
+++ b/tradein-mvp/backend/app/services/scrapers/cian.py
@@ -21,6 +21,7 @@ from datetime import UTC, datetime
 from typing import Any
 from urllib.parse import urlencode
 
+import sentry_sdk
 from curl_cffi.requests import AsyncSession
 
 from app.core.config import settings
@@ -222,6 +223,26 @@ class CianScraper(BaseScraper):
             lot = self._offer_to_lot(offer)
             if lot is not None:
                 lots.append(lot)
+
+        raw_count = len(offers_data)
+        saved_count = len(lots)
+        # Охрана от silent-failure: если получили offers, но ни один не прошёл парсинг
+        # — likely schema regression (как descriptionMinhash str→list[int] 2026-05-23).
+        if raw_count > 0 and saved_count == 0:
+            logger.error(
+                "cian SERP: 0/%d offers прошли _offer_to_lot — возможна schema regression",
+                raw_count,
+            )
+            try:
+                if settings.glitchtip_dsn:
+                    sentry_sdk.capture_message(
+                        f"cian SERP: {raw_count}/{raw_count} offers failed _offer_to_lot"
+                        " — possible schema regression",
+                        level="error",
+                    )
+            except Exception:
+                pass  # sentry_sdk not initialised in dev
+
         return lots
 
     def _offer_to_lot(self, offer: dict[str, Any]) -> ScrapedLot | None:
@@ -378,8 +399,16 @@ class CianScraper(BaseScraper):
 
             # ── Description + minhash ─────────────────────────────────────────
             description: str | None = offer.get("description")
-            # Cian предоставляет готовый minhash для dedup и cross-source matching
-            description_minhash: str | None = offer.get("descriptionMinhash")
+            # Cian предоставляет готовый minhash для dedup и cross-source matching.
+            # С ~2026-05-23 Cian изменил тип descriptionMinhash: str → list[int].
+            # Нормализуем оба варианта; НЕ sorted() — порядок band-ов значим для LSH.
+            _raw_minhash = offer.get("descriptionMinhash")
+            if isinstance(_raw_minhash, list):
+                description_minhash: str | None = ",".join(str(x) for x in _raw_minhash) or None
+            elif isinstance(_raw_minhash, str):
+                description_minhash = _raw_minhash or None
+            else:
+                description_minhash = None
             # Если Cian не дал minhash — вычисляем простой SHA1 из description
             if not description_minhash and description:
                 description_minhash = hashlib.sha1(
diff --git a/tradein-mvp/backend/tests/test_cian_serp_scraper.py b/tradein-mvp/backend/tests/test_cian_serp_scraper.py
index 76a9567e..a12a4029 100644
--- a/tradein-mvp/backend/tests/test_cian_serp_scraper.py
+++ b/tradein-mvp/backend/tests/test_cian_serp_scraper.py
@@ -5,11 +5,11 @@ Fixture формирует минимальный JS push() conforming to _cianC
 """
 
 import json
+
 import pytest
 
 from app.services.scrapers.cian import CianScraper, _format_address
 
-
 # ── Fixture helpers ──────────────────────────────────────────────────────────
 
 
@@ -51,7 +51,7 @@ def _make_full_offer(
     cadastral_number: str = "66:41:0204016:1234",
     building_cadastral_number: str = "66:41:0204016:100",
     description: str = "Продаётся отличная квартира",
-    description_minhash: str = "abc123def456abc1",
+    description_minhash: str | list[int] = "abc123def456abc1",
     newbuilding_id: int = 0,
     has_phones: bool = True,
     is_homeowner: bool = True,
@@ -240,6 +240,28 @@ def test_cian_serp_description_minhash_fallback():
     assert len(lot.description_minhash) == 32  # SHA1 hex [:32]
 
 
+def test_cian_serp_description_minhash_list_int():
+    """descriptionMinhash как list[int] (новый формат Cian ~2026-05-23) → строка через запятую.
+
+    Порядок элементов сохраняется как-есть — sorted() НЕ применяется,
+    т.к. порядок LSH-band'ов значим для dedup/cross-source matching.
+    Offer должен быть принят (не отброшен) — возвращает ScrapedLot, не None.
+    """
+    minhash_list = [8439545, 5781654, 555882, 407717]
+    offer = _make_full_offer(
+        description_minhash=minhash_list,  # type: ignore[arg-type]
+        description="Продаётся квартира с видом на парк",
+    )
+    html = _make_cian_serp_html([offer])
+
+    scraper = CianScraper()
+    lots = scraper._parse_serp_html(html)
+
+    assert len(lots) == 1, "offer с list[int] minhash должен быть принят, не отброшен"
+    lot = lots[0]
+    assert lot.description_minhash == "8439545,5781654,555882,407717"
+
+
 def test_cian_serp_phones():
     """phones — список dicts из state."""
     offer = _make_full_offer(has_phones=True)