fix(yandex-valuation): use curl_cffi chrome120 — Yandex gates SSR on Chrome TLS

Live probe today: plain httpx GET to /otsenka-kvartiry-po-adresu-onlayn/
returns 200 OK with full HTML, but body contains only the table header row
(__head). Data rows are CSR-rendered. curl_cffi with impersonate=chrome120
gets full SSR (21 rows, 133 cells, 38 dates verified).

Override __aenter__/__aexit__/_http_get only in YandexValuationScraper —
BaseScraper untouched (other scrapers don't need Chrome TLS).

Sibling pattern: yandex_realty.py + scripts/local-sweep-ekb-yandex.py
already use curl_cffi chrome120. This brings yandex_valuation in line.

Net effect: DOM-based parser from PR #544 will actually have DOM rows to
parse. Previous re-sweep returned 0 saves because rows weren't in SSR.
This commit is contained in:
lekss361 2026-05-24 19:37:30 +03:00
parent 8a88ea6508
commit ed6e52f393

View file

@ -22,6 +22,7 @@ from datetime import date
from typing import Any
from urllib.parse import urlencode
from curl_cffi.requests import AsyncSession as _CurlCffiSession
from pydantic import BaseModel, Field
from selectolax.parser import HTMLParser
@ -158,6 +159,31 @@ class YandexValuationScraper(BaseScraper):
def __init__(self) -> None:
super().__init__()
self.request_delay_sec = get_scraper_delay(self.name)
self._cffi_session: _CurlCffiSession | None = None
async def __aenter__(self) -> YandexValuationScraper: # type: ignore[override]
# Override: Yandex valuation endpoint gates SSR data on Chrome TLS
# fingerprint. Plain httpx returns shell HTML (CSR-only).
# Sibling: yandex_realty.py / scripts/local-sweep-ekb-yandex.py
self._cffi_session = _CurlCffiSession(impersonate="chrome120")
return self
async def __aexit__(self, *args: object) -> None: # type: ignore[override]
if self._cffi_session is not None:
await self._cffi_session.close()
self._cffi_session = None
async def _http_get(self, url: str, **kwargs: object) -> object: # type: ignore[override]
"""curl_cffi-based GET with Chrome120 impersonation.
Returns curl_cffi Response (compatible API: .status_code, .text).
Caller must check status_code; no automatic retry (BaseScraper.retry
decorator is per-method and not inherited cleanly when overridden).
"""
if self._cffi_session is None:
raise RuntimeError("YandexValuationScraper must be used as async context manager")
kwargs.setdefault("timeout", 30)
return await self._cffi_session.get(url, **kwargs)
async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override]
raise NotImplementedError(