From 3c2e03448d7befe4f8a8ffe1a216e7578220e90b Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 31 May 2026 08:54:25 +0300 Subject: [PATCH] fix(tradein): cian SERP descriptionMinhash list + citywide break-on-empty (T1+T2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit T1 (already in main via #840): normalize descriptionMinhash list[int]→str T2: warm-up GET / in __aenter__ to seed _CIAN_GK cookies before SERP requests; break-on-empty-page was already present in fetch_around_multi_room --- .../backend/app/services/scrapers/cian.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tradein-mvp/backend/app/services/scrapers/cian.py b/tradein-mvp/backend/app/services/scrapers/cian.py index 505ea902..cbace45b 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian.py +++ b/tradein-mvp/backend/app/services/scrapers/cian.py @@ -84,6 +84,25 @@ class CianScraper(BaseScraper): "Upgrade-Insecure-Requests": "1", }, ) + # Warm-up: посетить homepage чтобы получить сессионные cookies (_CIAN_GK/_yasc) + # и не выглядеть как наглый бот (первый запрос сразу на /cat.php подозрителен). + # Failure не критична — SERP-запросы продолжатся в любом случае. + try: + _warm = await self._cffi.get(f"{self.base_url}/") + logger.info( + "cian warm-up GET / → HTTP %d (cookies=%d)", + _warm.status_code, + len(list(self._cffi.cookies.jar)), + ) + # Последующие SERP-запросы выглядят как in-site navigation (Referer = /) + self._cffi.headers.update( + { + "Referer": f"{self.base_url}/", + "Sec-Fetch-Site": "same-origin", + } + ) + except Exception: + logger.warning("cian warm-up GET / failed — continuing without seed cookies") return self async def __aexit__(self, *args: Any) -> None: -- 2.45.3