fix(yandex-valuation): use curl_cffi chrome120 — Yandex gates SSR on Chrome TLS
Live probe today: plain httpx GET to /otsenka-kvartiry-po-adresu-onlayn/ returns 200 OK with full HTML, but body contains only the table header row (__head). Data rows are CSR-rendered. curl_cffi with impersonate=chrome120 gets full SSR (21 rows, 133 cells, 38 dates verified). Override __aenter__/__aexit__/_http_get only in YandexValuationScraper — BaseScraper untouched (other scrapers don't need Chrome TLS). Sibling pattern: yandex_realty.py + scripts/local-sweep-ekb-yandex.py already use curl_cffi chrome120. This brings yandex_valuation in line. Net effect: DOM-based parser from PR #544 will actually have DOM rows to parse. Previous re-sweep returned 0 saves because rows weren't in SSR.
This commit is contained in:
parent
8a88ea6508
commit
ed6e52f393
1 changed files with 26 additions and 0 deletions
|
|
@ -22,6 +22,7 @@ from datetime import date
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode
|
||||||
|
|
||||||
|
from curl_cffi.requests import AsyncSession as _CurlCffiSession
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
from selectolax.parser import HTMLParser
|
from selectolax.parser import HTMLParser
|
||||||
|
|
||||||
|
|
@ -158,6 +159,31 @@ class YandexValuationScraper(BaseScraper):
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.request_delay_sec = get_scraper_delay(self.name)
|
self.request_delay_sec = get_scraper_delay(self.name)
|
||||||
|
self._cffi_session: _CurlCffiSession | None = None
|
||||||
|
|
||||||
|
async def __aenter__(self) -> YandexValuationScraper: # type: ignore[override]
|
||||||
|
# Override: Yandex valuation endpoint gates SSR data on Chrome TLS
|
||||||
|
# fingerprint. Plain httpx returns shell HTML (CSR-only).
|
||||||
|
# Sibling: yandex_realty.py / scripts/local-sweep-ekb-yandex.py
|
||||||
|
self._cffi_session = _CurlCffiSession(impersonate="chrome120")
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, *args: object) -> None: # type: ignore[override]
|
||||||
|
if self._cffi_session is not None:
|
||||||
|
await self._cffi_session.close()
|
||||||
|
self._cffi_session = None
|
||||||
|
|
||||||
|
async def _http_get(self, url: str, **kwargs: object) -> object: # type: ignore[override]
|
||||||
|
"""curl_cffi-based GET with Chrome120 impersonation.
|
||||||
|
|
||||||
|
Returns curl_cffi Response (compatible API: .status_code, .text).
|
||||||
|
Caller must check status_code; no automatic retry (BaseScraper.retry
|
||||||
|
decorator is per-method and not inherited cleanly when overridden).
|
||||||
|
"""
|
||||||
|
if self._cffi_session is None:
|
||||||
|
raise RuntimeError("YandexValuationScraper must be used as async context manager")
|
||||||
|
kwargs.setdefault("timeout", 30)
|
||||||
|
return await self._cffi_session.get(url, **kwargs)
|
||||||
|
|
||||||
async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override]
|
async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override]
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue