fix(tradein): cian SERP descriptionMinhash list + citywide break-on-empty (T1+T2)
T1 (already in main via #840): normalize descriptionMinhash list[int]→str T2: warm-up GET / in __aenter__ to seed _CIAN_GK cookies before SERP requests; break-on-empty-page was already present in fetch_around_multi_room
This commit is contained in:
parent
de6afc99fa
commit
3c2e03448d
1 changed files with 19 additions and 0 deletions
|
|
@ -84,6 +84,25 @@ class CianScraper(BaseScraper):
|
||||||
"Upgrade-Insecure-Requests": "1",
|
"Upgrade-Insecure-Requests": "1",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
# Warm-up: посетить homepage чтобы получить сессионные cookies (_CIAN_GK/_yasc)
|
||||||
|
# и не выглядеть как наглый бот (первый запрос сразу на /cat.php подозрителен).
|
||||||
|
# Failure не критична — SERP-запросы продолжатся в любом случае.
|
||||||
|
try:
|
||||||
|
_warm = await self._cffi.get(f"{self.base_url}/")
|
||||||
|
logger.info(
|
||||||
|
"cian warm-up GET / → HTTP %d (cookies=%d)",
|
||||||
|
_warm.status_code,
|
||||||
|
len(list(self._cffi.cookies.jar)),
|
||||||
|
)
|
||||||
|
# Последующие SERP-запросы выглядят как in-site navigation (Referer = /)
|
||||||
|
self._cffi.headers.update(
|
||||||
|
{
|
||||||
|
"Referer": f"{self.base_url}/",
|
||||||
|
"Sec-Fetch-Site": "same-origin",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.warning("cian warm-up GET / failed — continuing without seed cookies")
|
||||||
return self
|
return self
|
||||||
|
|
||||||
async def __aexit__(self, *args: Any) -> None:
|
async def __aexit__(self, *args: Any) -> None:
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue