fix(scrapers): normalize cian descriptionMinhash list[int]→str + silent-0 guard (Refs #836) #840

Merged
bot-reviewer merged 1 commit from fix/836-cian-minhash into main 2026-05-30 20:28:57 +00:00
2 changed files with 55 additions and 4 deletions

View file

@ -21,6 +21,7 @@ from datetime import UTC, datetime
from typing import Any
from urllib.parse import urlencode
import sentry_sdk
from curl_cffi.requests import AsyncSession
from app.core.config import settings
@ -222,6 +223,26 @@ class CianScraper(BaseScraper):
lot = self._offer_to_lot(offer)
if lot is not None:
lots.append(lot)
raw_count = len(offers_data)
saved_count = len(lots)
# Охрана от silent-failure: если получили offers, но ни один не прошёл парсинг
# — likely schema regression (как descriptionMinhash str→list[int] 2026-05-23).
if raw_count > 0 and saved_count == 0:
logger.error(
"cian SERP: 0/%d offers прошли _offer_to_lot — возможна schema regression",
raw_count,
)
try:
if settings.glitchtip_dsn:
sentry_sdk.capture_message(
f"cian SERP: {raw_count}/{raw_count} offers failed _offer_to_lot"
" — possible schema regression",
level="error",
)
except Exception:
pass # sentry_sdk not initialised in dev
return lots
def _offer_to_lot(self, offer: dict[str, Any]) -> ScrapedLot | None:
@ -378,8 +399,16 @@ class CianScraper(BaseScraper):
# ── Description + minhash ─────────────────────────────────────────
description: str | None = offer.get("description")
# Cian предоставляет готовый minhash для dedup и cross-source matching
description_minhash: str | None = offer.get("descriptionMinhash")
# Cian предоставляет готовый minhash для dedup и cross-source matching.
# С ~2026-05-23 Cian изменил тип descriptionMinhash: str → list[int].
# Нормализуем оба варианта; НЕ sorted() — порядок band-ов значим для LSH.
_raw_minhash = offer.get("descriptionMinhash")
if isinstance(_raw_minhash, list):
description_minhash: str | None = ",".join(str(x) for x in _raw_minhash) or None
elif isinstance(_raw_minhash, str):
description_minhash = _raw_minhash or None
else:
description_minhash = None
# Если Cian не дал minhash — вычисляем простой SHA1 из description
if not description_minhash and description:
description_minhash = hashlib.sha1(

View file

@ -5,11 +5,11 @@ Fixture формирует минимальный JS push() conforming to _cianC
"""
import json
import pytest
from app.services.scrapers.cian import CianScraper, _format_address
# ── Fixture helpers ──────────────────────────────────────────────────────────
@ -51,7 +51,7 @@ def _make_full_offer(
cadastral_number: str = "66:41:0204016:1234",
building_cadastral_number: str = "66:41:0204016:100",
description: str = "Продаётся отличная квартира",
description_minhash: str = "abc123def456abc1",
description_minhash: str | list[int] = "abc123def456abc1",
newbuilding_id: int = 0,
has_phones: bool = True,
is_homeowner: bool = True,
@ -240,6 +240,28 @@ def test_cian_serp_description_minhash_fallback():
assert len(lot.description_minhash) == 32 # SHA1 hex [:32]
def test_cian_serp_description_minhash_list_int():
"""descriptionMinhash как list[int] (новый формат Cian ~2026-05-23) → строка через запятую.
Порядок элементов сохраняется как-есть sorted() НЕ применяется,
т.к. порядок LSH-band'ов значим для dedup/cross-source matching.
Offer должен быть принят (не отброшен) возвращает ScrapedLot, не None.
"""
minhash_list = [8439545, 5781654, 555882, 407717]
offer = _make_full_offer(
description_minhash=minhash_list, # type: ignore[arg-type]
description="Продаётся квартира с видом на парк",
)
html = _make_cian_serp_html([offer])
scraper = CianScraper()
lots = scraper._parse_serp_html(html)
assert len(lots) == 1, "offer с list[int] minhash должен быть принят, не отброшен"
lot = lots[0]
assert lot.description_minhash == "8439545,5781654,555882,407717"
def test_cian_serp_phones():
"""phones — список dicts из state."""
offer = _make_full_offer(has_phones=True)