fix(scrapers): normalize cian descriptionMinhash list[int]→str + silent-0 guard (Refs #836) #840
2 changed files with 55 additions and 4 deletions
|
|
@ -21,6 +21,7 @@ from datetime import UTC, datetime
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode
|
||||||
|
|
||||||
|
import sentry_sdk
|
||||||
from curl_cffi.requests import AsyncSession
|
from curl_cffi.requests import AsyncSession
|
||||||
|
|
||||||
from app.core.config import settings
|
from app.core.config import settings
|
||||||
|
|
@ -222,6 +223,26 @@ class CianScraper(BaseScraper):
|
||||||
lot = self._offer_to_lot(offer)
|
lot = self._offer_to_lot(offer)
|
||||||
if lot is not None:
|
if lot is not None:
|
||||||
lots.append(lot)
|
lots.append(lot)
|
||||||
|
|
||||||
|
raw_count = len(offers_data)
|
||||||
|
saved_count = len(lots)
|
||||||
|
# Охрана от silent-failure: если получили offers, но ни один не прошёл парсинг
|
||||||
|
# — likely schema regression (как descriptionMinhash str→list[int] 2026-05-23).
|
||||||
|
if raw_count > 0 and saved_count == 0:
|
||||||
|
logger.error(
|
||||||
|
"cian SERP: 0/%d offers прошли _offer_to_lot — возможна schema regression",
|
||||||
|
raw_count,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
if settings.glitchtip_dsn:
|
||||||
|
sentry_sdk.capture_message(
|
||||||
|
f"cian SERP: {raw_count}/{raw_count} offers failed _offer_to_lot"
|
||||||
|
" — possible schema regression",
|
||||||
|
level="error",
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
pass # sentry_sdk not initialised in dev
|
||||||
|
|
||||||
return lots
|
return lots
|
||||||
|
|
||||||
def _offer_to_lot(self, offer: dict[str, Any]) -> ScrapedLot | None:
|
def _offer_to_lot(self, offer: dict[str, Any]) -> ScrapedLot | None:
|
||||||
|
|
@ -378,8 +399,16 @@ class CianScraper(BaseScraper):
|
||||||
|
|
||||||
# ── Description + minhash ─────────────────────────────────────────
|
# ── Description + minhash ─────────────────────────────────────────
|
||||||
description: str | None = offer.get("description")
|
description: str | None = offer.get("description")
|
||||||
# Cian предоставляет готовый minhash для dedup и cross-source matching
|
# Cian предоставляет готовый minhash для dedup и cross-source matching.
|
||||||
description_minhash: str | None = offer.get("descriptionMinhash")
|
# С ~2026-05-23 Cian изменил тип descriptionMinhash: str → list[int].
|
||||||
|
# Нормализуем оба варианта; НЕ sorted() — порядок band-ов значим для LSH.
|
||||||
|
_raw_minhash = offer.get("descriptionMinhash")
|
||||||
|
if isinstance(_raw_minhash, list):
|
||||||
|
description_minhash: str | None = ",".join(str(x) for x in _raw_minhash) or None
|
||||||
|
elif isinstance(_raw_minhash, str):
|
||||||
|
description_minhash = _raw_minhash or None
|
||||||
|
else:
|
||||||
|
description_minhash = None
|
||||||
# Если Cian не дал minhash — вычисляем простой SHA1 из description
|
# Если Cian не дал minhash — вычисляем простой SHA1 из description
|
||||||
if not description_minhash and description:
|
if not description_minhash and description:
|
||||||
description_minhash = hashlib.sha1(
|
description_minhash = hashlib.sha1(
|
||||||
|
|
|
||||||
|
|
@ -5,11 +5,11 @@ Fixture формирует минимальный JS push() conforming to _cianC
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.services.scrapers.cian import CianScraper, _format_address
|
from app.services.scrapers.cian import CianScraper, _format_address
|
||||||
|
|
||||||
|
|
||||||
# ── Fixture helpers ──────────────────────────────────────────────────────────
|
# ── Fixture helpers ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -51,7 +51,7 @@ def _make_full_offer(
|
||||||
cadastral_number: str = "66:41:0204016:1234",
|
cadastral_number: str = "66:41:0204016:1234",
|
||||||
building_cadastral_number: str = "66:41:0204016:100",
|
building_cadastral_number: str = "66:41:0204016:100",
|
||||||
description: str = "Продаётся отличная квартира",
|
description: str = "Продаётся отличная квартира",
|
||||||
description_minhash: str = "abc123def456abc1",
|
description_minhash: str | list[int] = "abc123def456abc1",
|
||||||
newbuilding_id: int = 0,
|
newbuilding_id: int = 0,
|
||||||
has_phones: bool = True,
|
has_phones: bool = True,
|
||||||
is_homeowner: bool = True,
|
is_homeowner: bool = True,
|
||||||
|
|
@ -240,6 +240,28 @@ def test_cian_serp_description_minhash_fallback():
|
||||||
assert len(lot.description_minhash) == 32 # SHA1 hex [:32]
|
assert len(lot.description_minhash) == 32 # SHA1 hex [:32]
|
||||||
|
|
||||||
|
|
||||||
|
def test_cian_serp_description_minhash_list_int():
|
||||||
|
"""descriptionMinhash как list[int] (новый формат Cian ~2026-05-23) → строка через запятую.
|
||||||
|
|
||||||
|
Порядок элементов сохраняется как-есть — sorted() НЕ применяется,
|
||||||
|
т.к. порядок LSH-band'ов значим для dedup/cross-source matching.
|
||||||
|
Offer должен быть принят (не отброшен) — возвращает ScrapedLot, не None.
|
||||||
|
"""
|
||||||
|
minhash_list = [8439545, 5781654, 555882, 407717]
|
||||||
|
offer = _make_full_offer(
|
||||||
|
description_minhash=minhash_list, # type: ignore[arg-type]
|
||||||
|
description="Продаётся квартира с видом на парк",
|
||||||
|
)
|
||||||
|
html = _make_cian_serp_html([offer])
|
||||||
|
|
||||||
|
scraper = CianScraper()
|
||||||
|
lots = scraper._parse_serp_html(html)
|
||||||
|
|
||||||
|
assert len(lots) == 1, "offer с list[int] minhash должен быть принят, не отброшен"
|
||||||
|
lot = lots[0]
|
||||||
|
assert lot.description_minhash == "8439545,5781654,555882,407717"
|
||||||
|
|
||||||
|
|
||||||
def test_cian_serp_phones():
|
def test_cian_serp_phones():
|
||||||
"""phones — список dicts из state."""
|
"""phones — список dicts из state."""
|
||||||
offer = _make_full_offer(has_phones=True)
|
offer = _make_full_offer(has_phones=True)
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue