fix(tradein): cian SERP descriptionMinhash list + citywide break-on-empty (T1+T2)
T1 (already in main via #840): normalize descriptionMinhash list[int]→str T2: warm-up GET / in __aenter__ to seed _CIAN_GK cookies before SERP requests; break-on-empty-page was already present in fetch_around_multi_room
This commit is contained in:
parent
de6afc99fa
commit
3c2e03448d
1 changed files with 19 additions and 0 deletions
|
|
@ -84,6 +84,25 @@ class CianScraper(BaseScraper):
|
|||
"Upgrade-Insecure-Requests": "1",
|
||||
},
|
||||
)
|
||||
# Warm-up: посетить homepage чтобы получить сессионные cookies (_CIAN_GK/_yasc)
|
||||
# и не выглядеть как наглый бот (первый запрос сразу на /cat.php подозрителен).
|
||||
# Failure не критична — SERP-запросы продолжатся в любом случае.
|
||||
try:
|
||||
_warm = await self._cffi.get(f"{self.base_url}/")
|
||||
logger.info(
|
||||
"cian warm-up GET / → HTTP %d (cookies=%d)",
|
||||
_warm.status_code,
|
||||
len(list(self._cffi.cookies.jar)),
|
||||
)
|
||||
# Последующие SERP-запросы выглядят как in-site navigation (Referer = /)
|
||||
self._cffi.headers.update(
|
||||
{
|
||||
"Referer": f"{self.base_url}/",
|
||||
"Sec-Fetch-Site": "same-origin",
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("cian warm-up GET / failed — continuing without seed cookies")
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args: Any) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue