From e1c5f2184cd5e3f00d29e56df9f2bede2c24cf6f Mon Sep 17 00:00:00 2001 From: lekss361 Date: Fri, 3 Jul 2026 06:36:56 +0000 Subject: [PATCH] fix(tradein/scraper-kit): wire browser proxy-pool into yandex/cian SERP + persist activation flags (#2160) (#2268) --- .../backend/tests/test_kit_serp_proxy_pool.py | 164 ++++++++++++++++++ tradein-mvp/docker-compose.prod.yml | 5 + .../src/scraper_kit/orchestration/pipeline.py | 10 +- .../scraper_kit/orchestration/scheduler.py | 1 + .../src/scraper_kit/providers/cian/serp.py | 11 +- .../src/scraper_kit/providers/yandex/serp.py | 8 +- 6 files changed, 192 insertions(+), 7 deletions(-) create mode 100644 tradein-mvp/backend/tests/test_kit_serp_proxy_pool.py diff --git a/tradein-mvp/backend/tests/test_kit_serp_proxy_pool.py b/tradein-mvp/backend/tests/test_kit_serp_proxy_pool.py new file mode 100644 index 00000000..812c2955 --- /dev/null +++ b/tradein-mvp/backend/tests/test_kit_serp_proxy_pool.py @@ -0,0 +1,164 @@ +"""Тесты проброса browser-пула в SERP-скраперы yandex/cian (#2160). + +Зеркало P4 (#2164) для avito: там пул раздавался только в avito-путях, а yandex/cian +SERP создавали BrowserFetcher сами в __aenter__ без пула → весь трафик шёл через +env-прокси браузера. Здесь проверяем, что YandexRealtyScraper / CianScraper прокидывают +proxy_provider + use_pool=config.use_proxy_pool_browser в BrowserFetcher. + +Инвариант ship-dark: + - proxy_provider=None ИЛИ флаг off → BrowserFetcher(..., proxy_provider=None, use_pool=False), + т.е. env-прокси браузера, поведение байт-в-байт как до фикса. + - proxy_provider задан + флаг on → BrowserFetcher получает провайдер + use_pool=True. + +BrowserFetcher монки-патчится на recording-заглушку в модуле каждого скрапера, так что +реального HTTP к tradein-browser не происходит. +""" + +from __future__ import annotations + +import types +from typing import Any + +import pytest +from scraper_kit.providers.cian import serp as cian_serp +from scraper_kit.providers.yandex import serp as yandex_serp + + +class _FakeProxyProvider: + """ProxyProvider-заглушка: методы no-op (в этих тестах пул не вызывается — + BrowserFetcher замокан, acquire/release не срабатывают).""" + + def acquire(self, provider: str) -> None: + return None + + def release(self, lease: object) -> None: + return None + + def mark_health(self, lease: object, ok: bool) -> None: + return None + + +def _make_recorder() -> tuple[type, list[dict[str, Any]]]: + """Recording-заглушка BrowserFetcher: пишет init-kwargs в общий список.""" + calls: list[dict[str, Any]] = [] + + class _RecFetcher: + def __init__( + self, + source: str = "avito", + fetch_timeout_s: float = 120.0, + *, + endpoint: str, + proxy_provider: object | None = None, + use_pool: bool = False, + ) -> None: + calls.append( + { + "source": source, + "endpoint": endpoint, + "proxy_provider": proxy_provider, + "use_pool": use_pool, + } + ) + + async def __aenter__(self) -> _RecFetcher: + return self + + async def __aexit__(self, *args: object) -> None: + return None + + return _RecFetcher, calls + + +def _config(*, use_pool: bool) -> types.SimpleNamespace: + return types.SimpleNamespace( + browser_http_endpoint="http://browser:3000", + use_proxy_pool_browser=use_pool, + ) + + +# ── Yandex SERP ────────────────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_yandex_serp_wires_pool_when_flag_on(monkeypatch: pytest.MonkeyPatch) -> None: + """proxy_provider + флаг on → BrowserFetcher получает провайдер + use_pool=True.""" + rec_fetcher, calls = _make_recorder() + monkeypatch.setattr(yandex_serp, "BrowserFetcher", rec_fetcher) + provider = _FakeProxyProvider() + + scraper = yandex_serp.YandexRealtyScraper(_config(use_pool=True), proxy_provider=provider) + async with scraper: + pass + + assert len(calls) == 1 + assert calls[0]["source"] == "yandex" + assert calls[0]["proxy_provider"] is provider + assert calls[0]["use_pool"] is True + + +@pytest.mark.asyncio +async def test_yandex_serp_no_pool_by_default(monkeypatch: pytest.MonkeyPatch) -> None: + """Без provider/флага → BrowserFetcher(proxy_provider=None, use_pool=False) — ship-dark.""" + rec_fetcher, calls = _make_recorder() + monkeypatch.setattr(yandex_serp, "BrowserFetcher", rec_fetcher) + + scraper = yandex_serp.YandexRealtyScraper(_config(use_pool=False)) + async with scraper: + pass + + assert len(calls) == 1 + assert calls[0]["proxy_provider"] is None + assert calls[0]["use_pool"] is False + + +@pytest.mark.asyncio +async def test_yandex_serp_provider_without_flag_stays_dark( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """provider задан, но флаг off → use_pool=False (BrowserFetcher пул не трогает).""" + rec_fetcher, calls = _make_recorder() + monkeypatch.setattr(yandex_serp, "BrowserFetcher", rec_fetcher) + provider = _FakeProxyProvider() + + scraper = yandex_serp.YandexRealtyScraper(_config(use_pool=False), proxy_provider=provider) + async with scraper: + pass + + assert calls[0]["proxy_provider"] is provider + assert calls[0]["use_pool"] is False + + +# ── Cian SERP ──────────────────────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_cian_serp_wires_pool_when_flag_on(monkeypatch: pytest.MonkeyPatch) -> None: + """proxy_provider + флаг on → BrowserFetcher получает провайдер + use_pool=True.""" + rec_fetcher, calls = _make_recorder() + monkeypatch.setattr(cian_serp, "BrowserFetcher", rec_fetcher) + provider = _FakeProxyProvider() + + scraper = cian_serp.CianScraper(_config(use_pool=True), proxy_provider=provider) + async with scraper: + pass + + assert len(calls) == 1 + assert calls[0]["source"] == "cian" + assert calls[0]["proxy_provider"] is provider + assert calls[0]["use_pool"] is True + + +@pytest.mark.asyncio +async def test_cian_serp_no_pool_by_default(monkeypatch: pytest.MonkeyPatch) -> None: + """Без provider/флага → BrowserFetcher(proxy_provider=None, use_pool=False) — ship-dark.""" + rec_fetcher, calls = _make_recorder() + monkeypatch.setattr(cian_serp, "BrowserFetcher", rec_fetcher) + + scraper = cian_serp.CianScraper(_config(use_pool=False)) + async with scraper: + pass + + assert len(calls) == 1 + assert calls[0]["proxy_provider"] is None + assert calls[0]["use_pool"] is False diff --git a/tradein-mvp/docker-compose.prod.yml b/tradein-mvp/docker-compose.prod.yml index a27b6b8e..c1ccdcd6 100644 --- a/tradein-mvp/docker-compose.prod.yml +++ b/tradein-mvp/docker-compose.prod.yml @@ -177,6 +177,11 @@ services: GENDESIGN_FDW_PASSWORD: "${GENDESIGN_FDW_PASSWORD:-}" COOKIE_ENCRYPTION_KEY: "${COOKIE_ENCRYPTION_KEY:-}" SCHEDULER_ENABLE: "true" + # Активация scraper-kit + proxy-pool (#2126/#2160): kit-scheduler вместо + # legacy sweep-путей; curl/browser-фетчи берут прокси из пула scrape_proxies. + USE_KIT_SCHEDULER: "true" + USE_PROXY_POOL_CURL: "true" + USE_PROXY_POOL_BROWSER: "true" depends_on: postgres: condition: service_healthy diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/pipeline.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/pipeline.py index 35655234..30784467 100644 --- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/pipeline.py +++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/pipeline.py @@ -1632,6 +1632,7 @@ async def run_yandex_city_sweep( matcher: HouseMatcher, enrichment: EnrichmentJobs, shutdown_requested: Callable[[], bool] = lambda: False, + proxy_provider: ProxyProvider | None = None, anchors: list[tuple[float, float, str]] | None = None, radius_m: int = 25000, pages_per_anchor: int = 2, @@ -1791,7 +1792,7 @@ async def run_yandex_city_sweep( counters.lots_fetched, ) - async with YandexRealtyScraper(config) as scraper: + async with YandexRealtyScraper(config, proxy_provider=proxy_provider) as scraper: await scraper.fetch_around_multi_room( _a_lat, _a_lon, @@ -2216,7 +2217,7 @@ async def run_cian_city_sweep( nonlocal anchor_lots, consecutive_failures, cian_rotations_done # ── Phase 1+2: SERP + save ───────────────────────────────── - async with CianScraper(config) as scraper: + async with CianScraper(config, proxy_provider=proxy_provider) as scraper: anchor_lots = await scraper.fetch_around_multi_room( _a_lat, _a_lon, @@ -2667,7 +2668,7 @@ async def run_cian_full_load( try: _hb_task = asyncio.create_task(_background_heartbeat()) - async with CianScraper(config) as scraper: + async with CianScraper(config, proxy_provider=proxy_provider) as scraper: scraper.request_delay_sec = request_delay_sec # #1949: per-fetch hard-cancel для browser-fetch'ей внутри bucket-gather. @@ -2865,6 +2866,7 @@ async def run_yandex_full_load( matcher: HouseMatcher, enrichment: EnrichmentJobs, shutdown_requested: Callable[[], bool] = lambda: False, + proxy_provider: ProxyProvider | None = None, price_cap_per_bucket: int = 500, request_delay_sec: float = 2.0, concurrency: int = 4, @@ -2965,7 +2967,7 @@ async def run_yandex_full_load( runs.update_heartbeat(db, run_id, counters.to_dict()) try: - async with YandexRealtyScraper(config) as scraper: + async with YandexRealtyScraper(config, proxy_provider=proxy_provider) as scraper: scraper.request_delay_sec = request_delay_sec await scraper.fetch_all_secondary( diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/scheduler.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/scheduler.py index e3c91e7e..5f9b1953 100644 --- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/scheduler.py +++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/scheduler.py @@ -520,6 +520,7 @@ async def _job_yandex_city_sweep( matcher=ctx.matcher, enrichment=ctx.enrichment, shutdown_requested=ctx.shutdown_requested, + proxy_provider=ctx.proxy_provider, pages_per_anchor=int(params.get("pages_per_anchor", 2)), request_delay_sec=float(params.get("request_delay_sec", 9.0)), radius_m=int(params.get("radius_m", 1500)), diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/serp.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/serp.py index 5c5679e7..54aa8fec 100644 --- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/serp.py +++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/serp.py @@ -46,7 +46,7 @@ from scraper_kit.repair_state_normalizer import ( ) if TYPE_CHECKING: - from scraper_kit.contracts import ScraperConfig + from scraper_kit.contracts import ProxyProvider, ScraperConfig logger = logging.getLogger(__name__) @@ -118,6 +118,7 @@ class CianScraper(BaseScraper): config: ScraperConfig, *, delay_provider: Callable[[str], float] | None = None, + proxy_provider: ProxyProvider | None = None, ) -> None: super().__init__() # Strangler-инжекция (#2133): конфиг и провайдер задержки приходят снаружи @@ -127,6 +128,9 @@ class CianScraper(BaseScraper): self._config = config if delay_provider is not None: self.request_delay_sec = delay_provider(self.name) + # #2160: пул прокси для camoufox-браузера (за флагом use_proxy_pool_browser). + # None → BrowserFetcher без пула = env-прокси браузера (ship-dark, как сейчас). + self._proxy_provider = proxy_provider self._browser: BrowserFetcher | None = None async def __aenter__(self) -> CianScraper: @@ -136,7 +140,10 @@ class CianScraper(BaseScraper): # отдельный браузер+мобильный прокси для cian. Прокси ротирует IP сам, поэтому # code-side warm-up cookies и changeip-ротация больше не нужны. self._browser = BrowserFetcher( - source="cian", endpoint=self._config.browser_http_endpoint + source="cian", + endpoint=self._config.browser_http_endpoint, + proxy_provider=self._proxy_provider, + use_pool=self._config.use_proxy_pool_browser, ) await self._browser.__aenter__() return self diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/yandex/serp.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/yandex/serp.py index e93b4985..94be04cc 100644 --- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/yandex/serp.py +++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/yandex/serp.py @@ -53,7 +53,7 @@ from scraper_kit.price_brackets import get_price_seed_brackets from scraper_kit.pricing import BisectionConfig, ProbeFailPolicy, ProbeResult, walk_price_range if TYPE_CHECKING: - from scraper_kit.contracts import ScraperConfig + from scraper_kit.contracts import ProxyProvider, ScraperConfig logger = logging.getLogger(__name__) @@ -461,6 +461,7 @@ class YandexRealtyScraper(BaseScraper): city: str = DEFAULT_CITY, *, delay_provider: Callable[[str], float] | None = None, + proxy_provider: ProxyProvider | None = None, ) -> None: super().__init__() self.city = city @@ -471,6 +472,9 @@ class YandexRealtyScraper(BaseScraper): self._config = config if delay_provider is not None: self.request_delay_sec = delay_provider(self.name) + # #2160: пул прокси для camoufox-браузера (за флагом use_proxy_pool_browser). + # None → BrowserFetcher без пула = env-прокси браузера (ship-dark, как сейчас). + self._proxy_provider = proxy_provider self._browser: BrowserFetcher | None = None # _cffi_session retained only for _rotate_ip (changeip call). self._cffi_session: _CurlCffiSession | None = None @@ -490,6 +494,8 @@ class YandexRealtyScraper(BaseScraper): source="yandex", fetch_timeout_s=_YANDEX_BROWSER_FETCH_TIMEOUT_S, endpoint=self._config.browser_http_endpoint, + proxy_provider=self._proxy_provider, + use_pool=self._config.use_proxy_pool_browser, ) await self._browser.__aenter__() return self