fix(scrapers): normalize cian descriptionMinhash list[int]→str + silent-0 guard (Refs #836) #840
2 changed files with 55 additions and 4 deletions
|
|
@ -21,6 +21,7 @@ from datetime import UTC, datetime
|
|||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import sentry_sdk
|
||||
from curl_cffi.requests import AsyncSession
|
||||
|
||||
from app.core.config import settings
|
||||
|
|
@ -222,6 +223,26 @@ class CianScraper(BaseScraper):
|
|||
lot = self._offer_to_lot(offer)
|
||||
if lot is not None:
|
||||
lots.append(lot)
|
||||
|
||||
raw_count = len(offers_data)
|
||||
saved_count = len(lots)
|
||||
# Охрана от silent-failure: если получили offers, но ни один не прошёл парсинг
|
||||
# — likely schema regression (как descriptionMinhash str→list[int] 2026-05-23).
|
||||
if raw_count > 0 and saved_count == 0:
|
||||
logger.error(
|
||||
"cian SERP: 0/%d offers прошли _offer_to_lot — возможна schema regression",
|
||||
raw_count,
|
||||
)
|
||||
try:
|
||||
if settings.glitchtip_dsn:
|
||||
sentry_sdk.capture_message(
|
||||
f"cian SERP: {raw_count}/{raw_count} offers failed _offer_to_lot"
|
||||
" — possible schema regression",
|
||||
level="error",
|
||||
)
|
||||
except Exception:
|
||||
pass # sentry_sdk not initialised in dev
|
||||
|
||||
return lots
|
||||
|
||||
def _offer_to_lot(self, offer: dict[str, Any]) -> ScrapedLot | None:
|
||||
|
|
@ -378,8 +399,16 @@ class CianScraper(BaseScraper):
|
|||
|
||||
# ── Description + minhash ─────────────────────────────────────────
|
||||
description: str | None = offer.get("description")
|
||||
# Cian предоставляет готовый minhash для dedup и cross-source matching
|
||||
description_minhash: str | None = offer.get("descriptionMinhash")
|
||||
# Cian предоставляет готовый minhash для dedup и cross-source matching.
|
||||
# С ~2026-05-23 Cian изменил тип descriptionMinhash: str → list[int].
|
||||
# Нормализуем оба варианта; НЕ sorted() — порядок band-ов значим для LSH.
|
||||
_raw_minhash = offer.get("descriptionMinhash")
|
||||
if isinstance(_raw_minhash, list):
|
||||
description_minhash: str | None = ",".join(str(x) for x in _raw_minhash) or None
|
||||
elif isinstance(_raw_minhash, str):
|
||||
description_minhash = _raw_minhash or None
|
||||
else:
|
||||
description_minhash = None
|
||||
# Если Cian не дал minhash — вычисляем простой SHA1 из description
|
||||
if not description_minhash and description:
|
||||
description_minhash = hashlib.sha1(
|
||||
|
|
|
|||
|
|
@ -5,11 +5,11 @@ Fixture формирует минимальный JS push() conforming to _cianC
|
|||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.cian import CianScraper, _format_address
|
||||
|
||||
|
||||
# ── Fixture helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
|
@ -51,7 +51,7 @@ def _make_full_offer(
|
|||
cadastral_number: str = "66:41:0204016:1234",
|
||||
building_cadastral_number: str = "66:41:0204016:100",
|
||||
description: str = "Продаётся отличная квартира",
|
||||
description_minhash: str = "abc123def456abc1",
|
||||
description_minhash: str | list[int] = "abc123def456abc1",
|
||||
newbuilding_id: int = 0,
|
||||
has_phones: bool = True,
|
||||
is_homeowner: bool = True,
|
||||
|
|
@ -240,6 +240,28 @@ def test_cian_serp_description_minhash_fallback():
|
|||
assert len(lot.description_minhash) == 32 # SHA1 hex [:32]
|
||||
|
||||
|
||||
def test_cian_serp_description_minhash_list_int():
|
||||
"""descriptionMinhash как list[int] (новый формат Cian ~2026-05-23) → строка через запятую.
|
||||
|
||||
Порядок элементов сохраняется как-есть — sorted() НЕ применяется,
|
||||
т.к. порядок LSH-band'ов значим для dedup/cross-source matching.
|
||||
Offer должен быть принят (не отброшен) — возвращает ScrapedLot, не None.
|
||||
"""
|
||||
minhash_list = [8439545, 5781654, 555882, 407717]
|
||||
offer = _make_full_offer(
|
||||
description_minhash=minhash_list, # type: ignore[arg-type]
|
||||
description="Продаётся квартира с видом на парк",
|
||||
)
|
||||
html = _make_cian_serp_html([offer])
|
||||
|
||||
scraper = CianScraper()
|
||||
lots = scraper._parse_serp_html(html)
|
||||
|
||||
assert len(lots) == 1, "offer с list[int] minhash должен быть принят, не отброшен"
|
||||
lot = lots[0]
|
||||
assert lot.description_minhash == "8439545,5781654,555882,407717"
|
||||
|
||||
|
||||
def test_cian_serp_phones():
|
||||
"""phones — список dicts из state."""
|
||||
offer = _make_full_offer(has_phones=True)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue