fix(tradein/scraper-kit): wire browser proxy-pool into yandex/cian SERP + persist activation flags (#2160) (#2268)

tradein-mvp/backend/tests/test_kit_serp_proxy_pool.py

@@ -0,0 +1,164 @@
+"""Тесты проброса browser-пула в SERP-скраперы yandex/cian (#2160).
+
+Зеркало P4 (#2164) для avito: там пул раздавался только в avito-путях, а yandex/cian
+SERP создавали BrowserFetcher сами в __aenter__ без пула → весь трафик шёл через
+env-прокси браузера. Здесь проверяем, что YandexRealtyScraper / CianScraper прокидывают
+proxy_provider + use_pool=config.use_proxy_pool_browser в BrowserFetcher.
+
+Инвариант ship-dark:
+  - proxy_provider=None ИЛИ флаг off → BrowserFetcher(..., proxy_provider=None, use_pool=False),
+    т.е. env-прокси браузера, поведение байт-в-байт как до фикса.
+  - proxy_provider задан + флаг on → BrowserFetcher получает провайдер + use_pool=True.
+
+BrowserFetcher монки-патчится на recording-заглушку в модуле каждого скрапера, так что
+реального HTTP к tradein-browser не происходит.
+"""
+
+from __future__ import annotations
+
+import types
+from typing import Any
+
+import pytest
+from scraper_kit.providers.cian import serp as cian_serp
+from scraper_kit.providers.yandex import serp as yandex_serp
+
+
+class _FakeProxyProvider:
+    """ProxyProvider-заглушка: методы no-op (в этих тестах пул не вызывается —
+    BrowserFetcher замокан, acquire/release не срабатывают)."""
+
+    def acquire(self, provider: str) -> None:
+        return None
+
+    def release(self, lease: object) -> None:
+        return None
+
+    def mark_health(self, lease: object, ok: bool) -> None:
+        return None
+
+
+def _make_recorder() -> tuple[type, list[dict[str, Any]]]:
+    """Recording-заглушка BrowserFetcher: пишет init-kwargs в общий список."""
+    calls: list[dict[str, Any]] = []
+
+    class _RecFetcher:
+        def __init__(
+            self,
+            source: str = "avito",
+            fetch_timeout_s: float = 120.0,
+            *,
+            endpoint: str,
+            proxy_provider: object | None = None,
+            use_pool: bool = False,
+        ) -> None:
+            calls.append(
+                {
+                    "source": source,
+                    "endpoint": endpoint,
+                    "proxy_provider": proxy_provider,
+                    "use_pool": use_pool,
+                }
+            )
+
+        async def __aenter__(self) -> _RecFetcher:
+            return self
+
+        async def __aexit__(self, *args: object) -> None:
+            return None
+
+    return _RecFetcher, calls
+
+
+def _config(*, use_pool: bool) -> types.SimpleNamespace:
+    return types.SimpleNamespace(
+        browser_http_endpoint="http://browser:3000",
+        use_proxy_pool_browser=use_pool,
+    )
+
+
+# ── Yandex SERP ──────────────────────────────────────────────────────────────
+
+
+@pytest.mark.asyncio
+async def test_yandex_serp_wires_pool_when_flag_on(monkeypatch: pytest.MonkeyPatch) -> None:
+    """proxy_provider + флаг on → BrowserFetcher получает провайдер + use_pool=True."""
+    rec_fetcher, calls = _make_recorder()
+    monkeypatch.setattr(yandex_serp, "BrowserFetcher", rec_fetcher)
+    provider = _FakeProxyProvider()
+
+    scraper = yandex_serp.YandexRealtyScraper(_config(use_pool=True), proxy_provider=provider)
+    async with scraper:
+        pass
+
+    assert len(calls) == 1
+    assert calls[0]["source"] == "yandex"
+    assert calls[0]["proxy_provider"] is provider
+    assert calls[0]["use_pool"] is True
+
+
+@pytest.mark.asyncio
+async def test_yandex_serp_no_pool_by_default(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Без provider/флага → BrowserFetcher(proxy_provider=None, use_pool=False) — ship-dark."""
+    rec_fetcher, calls = _make_recorder()
+    monkeypatch.setattr(yandex_serp, "BrowserFetcher", rec_fetcher)
+
+    scraper = yandex_serp.YandexRealtyScraper(_config(use_pool=False))
+    async with scraper:
+        pass
+
+    assert len(calls) == 1
+    assert calls[0]["proxy_provider"] is None
+    assert calls[0]["use_pool"] is False
+
+
+@pytest.mark.asyncio
+async def test_yandex_serp_provider_without_flag_stays_dark(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """provider задан, но флаг off → use_pool=False (BrowserFetcher пул не трогает)."""
+    rec_fetcher, calls = _make_recorder()
+    monkeypatch.setattr(yandex_serp, "BrowserFetcher", rec_fetcher)
+    provider = _FakeProxyProvider()
+
+    scraper = yandex_serp.YandexRealtyScraper(_config(use_pool=False), proxy_provider=provider)
+    async with scraper:
+        pass
+
+    assert calls[0]["proxy_provider"] is provider
+    assert calls[0]["use_pool"] is False
+
+
+# ── Cian SERP ────────────────────────────────────────────────────────────────
+
+
+@pytest.mark.asyncio
+async def test_cian_serp_wires_pool_when_flag_on(monkeypatch: pytest.MonkeyPatch) -> None:
+    """proxy_provider + флаг on → BrowserFetcher получает провайдер + use_pool=True."""
+    rec_fetcher, calls = _make_recorder()
+    monkeypatch.setattr(cian_serp, "BrowserFetcher", rec_fetcher)
+    provider = _FakeProxyProvider()
+
+    scraper = cian_serp.CianScraper(_config(use_pool=True), proxy_provider=provider)
+    async with scraper:
+        pass
+
+    assert len(calls) == 1
+    assert calls[0]["source"] == "cian"
+    assert calls[0]["proxy_provider"] is provider
+    assert calls[0]["use_pool"] is True
+
+
+@pytest.mark.asyncio
+async def test_cian_serp_no_pool_by_default(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Без provider/флага → BrowserFetcher(proxy_provider=None, use_pool=False) — ship-dark."""
+    rec_fetcher, calls = _make_recorder()
+    monkeypatch.setattr(cian_serp, "BrowserFetcher", rec_fetcher)
+
+    scraper = cian_serp.CianScraper(_config(use_pool=False))
+    async with scraper:
+        pass
+
+    assert len(calls) == 1
+    assert calls[0]["proxy_provider"] is None
+    assert calls[0]["use_pool"] is False

tradein-mvp/docker-compose.prod.yml

@@ -177,6 +177,11 @@ services:
       GENDESIGN_FDW_PASSWORD: "${GENDESIGN_FDW_PASSWORD:-}"
       COOKIE_ENCRYPTION_KEY: "${COOKIE_ENCRYPTION_KEY:-}"
       SCHEDULER_ENABLE: "true"
+      # Активация scraper-kit + proxy-pool (#2126/#2160): kit-scheduler вместо
+      # legacy sweep-путей; curl/browser-фетчи берут прокси из пула scrape_proxies.
+      USE_KIT_SCHEDULER: "true"
+      USE_PROXY_POOL_CURL: "true"
+      USE_PROXY_POOL_BROWSER: "true"
     depends_on:
       postgres:
         condition: service_healthy

tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/pipeline.py

@@ -1632,6 +1632,7 @@ async def run_yandex_city_sweep(
     matcher: HouseMatcher,
     enrichment: EnrichmentJobs,
     shutdown_requested: Callable[[], bool] = lambda: False,
+    proxy_provider: ProxyProvider | None = None,
     anchors: list[tuple[float, float, str]] | None = None,
     radius_m: int = 25000,
     pages_per_anchor: int = 2,
@@ -1791,7 +1792,7 @@ async def run_yandex_city_sweep(
                         counters.lots_fetched,
                     )
 
-                async with YandexRealtyScraper(config) as scraper:
+                async with YandexRealtyScraper(config, proxy_provider=proxy_provider) as scraper:
                     await scraper.fetch_around_multi_room(
                         _a_lat,
                         _a_lon,
@@ -2216,7 +2217,7 @@ async def run_cian_city_sweep(
                 nonlocal anchor_lots, consecutive_failures, cian_rotations_done
 
                 # ── Phase 1+2: SERP + save ─────────────────────────────────
-                async with CianScraper(config) as scraper:
+                async with CianScraper(config, proxy_provider=proxy_provider) as scraper:
                     anchor_lots = await scraper.fetch_around_multi_room(
                         _a_lat,
                         _a_lon,
@@ -2667,7 +2668,7 @@ async def run_cian_full_load(
     try:
         _hb_task = asyncio.create_task(_background_heartbeat())
 
-        async with CianScraper(config) as scraper:
+        async with CianScraper(config, proxy_provider=proxy_provider) as scraper:
             scraper.request_delay_sec = request_delay_sec
 
             # #1949: per-fetch hard-cancel для browser-fetch'ей внутри bucket-gather.
@@ -2865,6 +2866,7 @@ async def run_yandex_full_load(
     matcher: HouseMatcher,
     enrichment: EnrichmentJobs,
     shutdown_requested: Callable[[], bool] = lambda: False,
+    proxy_provider: ProxyProvider | None = None,
     price_cap_per_bucket: int = 500,
     request_delay_sec: float = 2.0,
     concurrency: int = 4,
@@ -2965,7 +2967,7 @@ async def run_yandex_full_load(
         runs.update_heartbeat(db, run_id, counters.to_dict())
 
     try:
-        async with YandexRealtyScraper(config) as scraper:
+        async with YandexRealtyScraper(config, proxy_provider=proxy_provider) as scraper:
             scraper.request_delay_sec = request_delay_sec
 
             await scraper.fetch_all_secondary(

tradein-mvp/packages/scraper-kit/src/scraper_kit/orchestration/scheduler.py

@@ -520,6 +520,7 @@ async def _job_yandex_city_sweep(
         matcher=ctx.matcher,
         enrichment=ctx.enrichment,
         shutdown_requested=ctx.shutdown_requested,
+        proxy_provider=ctx.proxy_provider,
         pages_per_anchor=int(params.get("pages_per_anchor", 2)),
         request_delay_sec=float(params.get("request_delay_sec", 9.0)),
         radius_m=int(params.get("radius_m", 1500)),

tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/serp.py

@@ -46,7 +46,7 @@ from scraper_kit.repair_state_normalizer import (
 )
 
 if TYPE_CHECKING:
-    from scraper_kit.contracts import ScraperConfig
+    from scraper_kit.contracts import ProxyProvider, ScraperConfig
 
 logger = logging.getLogger(__name__)
 
@@ -118,6 +118,7 @@ class CianScraper(BaseScraper):
         config: ScraperConfig,
         *,
         delay_provider: Callable[[str], float] | None = None,
+        proxy_provider: ProxyProvider | None = None,
     ) -> None:
         super().__init__()
         # Strangler-инжекция (#2133): конфиг и провайдер задержки приходят снаружи
@@ -127,6 +128,9 @@ class CianScraper(BaseScraper):
         self._config = config
         if delay_provider is not None:
             self.request_delay_sec = delay_provider(self.name)
+        # #2160: пул прокси для camoufox-браузера (за флагом use_proxy_pool_browser).
+        # None → BrowserFetcher без пула = env-прокси браузера (ship-dark, как сейчас).
+        self._proxy_provider = proxy_provider
         self._browser: BrowserFetcher | None = None
 
     async def __aenter__(self) -> CianScraper:
@@ -136,7 +140,10 @@ class CianScraper(BaseScraper):
         # отдельный браузер+мобильный прокси для cian. Прокси ротирует IP сам, поэтому
         # code-side warm-up cookies и changeip-ротация больше не нужны.
         self._browser = BrowserFetcher(
-            source="cian", endpoint=self._config.browser_http_endpoint
+            source="cian",
+            endpoint=self._config.browser_http_endpoint,
+            proxy_provider=self._proxy_provider,
+            use_pool=self._config.use_proxy_pool_browser,
         )
         await self._browser.__aenter__()
         return self

tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/yandex/serp.py

@@ -53,7 +53,7 @@ from scraper_kit.price_brackets import get_price_seed_brackets
 from scraper_kit.pricing import BisectionConfig, ProbeFailPolicy, ProbeResult, walk_price_range
 
 if TYPE_CHECKING:
-    from scraper_kit.contracts import ScraperConfig
+    from scraper_kit.contracts import ProxyProvider, ScraperConfig
 
 logger = logging.getLogger(__name__)
 
@@ -461,6 +461,7 @@ class YandexRealtyScraper(BaseScraper):
         city: str = DEFAULT_CITY,
         *,
         delay_provider: Callable[[str], float] | None = None,
+        proxy_provider: ProxyProvider | None = None,
     ) -> None:
         super().__init__()
         self.city = city
@@ -471,6 +472,9 @@ class YandexRealtyScraper(BaseScraper):
         self._config = config
         if delay_provider is not None:
             self.request_delay_sec = delay_provider(self.name)
+        # #2160: пул прокси для camoufox-браузера (за флагом use_proxy_pool_browser).
+        # None → BrowserFetcher без пула = env-прокси браузера (ship-dark, как сейчас).
+        self._proxy_provider = proxy_provider
         self._browser: BrowserFetcher | None = None
         # _cffi_session retained only for _rotate_ip (changeip call).
         self._cffi_session: _CurlCffiSession | None = None
@@ -490,6 +494,8 @@ class YandexRealtyScraper(BaseScraper):
             source="yandex",
             fetch_timeout_s=_YANDEX_BROWSER_FETCH_TIMEOUT_S,
             endpoint=self._config.browser_http_endpoint,
+            proxy_provider=self._proxy_provider,
+            use_pool=self._config.use_proxy_pool_browser,
         )
         await self._browser.__aenter__()
         return self