fix(yandex-valuation): curl_cffi chrome120 — Yandex gates SSR on Chrome TLS (#545)
This commit is contained in:
parent
8a88ea6508
commit
f7a5c5d4f7
1 changed files with 26 additions and 0 deletions
|
|
@ -22,6 +22,7 @@ from datetime import date
|
|||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from curl_cffi.requests import AsyncSession as _CurlCffiSession
|
||||
from pydantic import BaseModel, Field
|
||||
from selectolax.parser import HTMLParser
|
||||
|
||||
|
|
@ -158,6 +159,31 @@ class YandexValuationScraper(BaseScraper):
|
|||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.request_delay_sec = get_scraper_delay(self.name)
|
||||
self._cffi_session: _CurlCffiSession | None = None
|
||||
|
||||
async def __aenter__(self) -> YandexValuationScraper: # type: ignore[override]
|
||||
# Override: Yandex valuation endpoint gates SSR data on Chrome TLS
|
||||
# fingerprint. Plain httpx returns shell HTML (CSR-only).
|
||||
# Sibling: yandex_realty.py / scripts/local-sweep-ekb-yandex.py
|
||||
self._cffi_session = _CurlCffiSession(impersonate="chrome120")
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args: object) -> None: # type: ignore[override]
|
||||
if self._cffi_session is not None:
|
||||
await self._cffi_session.close()
|
||||
self._cffi_session = None
|
||||
|
||||
async def _http_get(self, url: str, **kwargs: object) -> object: # type: ignore[override]
|
||||
"""curl_cffi-based GET with Chrome120 impersonation.
|
||||
|
||||
Returns curl_cffi Response (compatible API: .status_code, .text).
|
||||
Caller must check status_code; no automatic retry (BaseScraper.retry
|
||||
decorator is per-method and not inherited cleanly when overridden).
|
||||
"""
|
||||
if self._cffi_session is None:
|
||||
raise RuntimeError("YandexValuationScraper must be used as async context manager")
|
||||
kwargs.setdefault("timeout", 30)
|
||||
return await self._cffi_session.get(url, **kwargs)
|
||||
|
||||
async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override]
|
||||
raise NotImplementedError(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue