From ed6e52f393b91041aa59f357f0673f62ae31604b Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 24 May 2026 19:37:30 +0300 Subject: [PATCH] =?UTF-8?q?fix(yandex-valuation):=20use=20curl=5Fcffi=20ch?= =?UTF-8?q?rome120=20=E2=80=94=20Yandex=20gates=20SSR=20on=20Chrome=20TLS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live probe today: plain httpx GET to /otsenka-kvartiry-po-adresu-onlayn/ returns 200 OK with full HTML, but body contains only the table header row (__head). Data rows are CSR-rendered. curl_cffi with impersonate=chrome120 gets full SSR (21 rows, 133 cells, 38 dates verified). Override __aenter__/__aexit__/_http_get only in YandexValuationScraper — BaseScraper untouched (other scrapers don't need Chrome TLS). Sibling pattern: yandex_realty.py + scripts/local-sweep-ekb-yandex.py already use curl_cffi chrome120. This brings yandex_valuation in line. Net effect: DOM-based parser from PR #544 will actually have DOM rows to parse. Previous re-sweep returned 0 saves because rows weren't in SSR. --- .../app/services/scrapers/yandex_valuation.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py b/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py index 467dc8bd..c4e38ff6 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py @@ -22,6 +22,7 @@ from datetime import date from typing import Any from urllib.parse import urlencode +from curl_cffi.requests import AsyncSession as _CurlCffiSession from pydantic import BaseModel, Field from selectolax.parser import HTMLParser @@ -158,6 +159,31 @@ class YandexValuationScraper(BaseScraper): def __init__(self) -> None: super().__init__() self.request_delay_sec = get_scraper_delay(self.name) + self._cffi_session: _CurlCffiSession | None = None + + async def __aenter__(self) -> YandexValuationScraper: # type: ignore[override] + # Override: Yandex valuation endpoint gates SSR data on Chrome TLS + # fingerprint. Plain httpx returns shell HTML (CSR-only). + # Sibling: yandex_realty.py / scripts/local-sweep-ekb-yandex.py + self._cffi_session = _CurlCffiSession(impersonate="chrome120") + return self + + async def __aexit__(self, *args: object) -> None: # type: ignore[override] + if self._cffi_session is not None: + await self._cffi_session.close() + self._cffi_session = None + + async def _http_get(self, url: str, **kwargs: object) -> object: # type: ignore[override] + """curl_cffi-based GET with Chrome120 impersonation. + + Returns curl_cffi Response (compatible API: .status_code, .text). + Caller must check status_code; no automatic retry (BaseScraper.retry + decorator is per-method and not inherited cleanly when overridden). + """ + if self._cffi_session is None: + raise RuntimeError("YandexValuationScraper must be used as async context manager") + kwargs.setdefault("timeout", 30) + return await self._cffi_session.get(url, **kwargs) async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override] raise NotImplementedError(