fix(tradein): cian SERP descriptionMinhash list + citywide break-on-empty (T1+T2)

T1 (already in main via #840): normalize descriptionMinhash list[int]→str
T2: warm-up GET / in __aenter__ to seed _CIAN_GK cookies before SERP requests;
break-on-empty-page was already present in fetch_around_multi_room
This commit is contained in:
lekss361 2026-05-31 08:54:25 +03:00
parent de6afc99fa
commit 3c2e03448d

View file

@ -84,6 +84,25 @@ class CianScraper(BaseScraper):
"Upgrade-Insecure-Requests": "1",
},
)
# Warm-up: посетить homepage чтобы получить сессионные cookies (_CIAN_GK/_yasc)
# и не выглядеть как наглый бот (первый запрос сразу на /cat.php подозрителен).
# Failure не критична — SERP-запросы продолжатся в любом случае.
try:
_warm = await self._cffi.get(f"{self.base_url}/")
logger.info(
"cian warm-up GET / → HTTP %d (cookies=%d)",
_warm.status_code,
len(list(self._cffi.cookies.jar)),
)
# Последующие SERP-запросы выглядят как in-site navigation (Referer = /)
self._cffi.headers.update(
{
"Referer": f"{self.base_url}/",
"Sec-Fetch-Site": "same-origin",
}
)
except Exception:
logger.warning("cian warm-up GET / failed — continuing without seed cookies")
return self
async def __aexit__(self, *args: Any) -> None: