fix(tradein/scraper-kit): wire browser proxy-pool into yandex/cian SERP paths (#2160)
P4 (#2164) подключил раздачу прокси из пула (scrape_proxies) в camoufox-браузер только для avito-путей. Yandex/Cian SERP-скраперы создавали BrowserFetcher сами в __aenter__ без пула → весь их трафик шёл через env-прокси браузера (перегружен). Прокидываем proxy_provider + use_pool=config.use_proxy_pool_browser в BrowserFetcher внутри YandexRealtyScraper / CianScraper, а сам proxy_provider — через run_yandex_city_sweep / run_yandex_full_load / run_cian_city_sweep / run_cian_full_load и kit-scheduler (_job_yandex_city_sweep). Ship-dark сохранён: proxy_provider=None ИЛИ флаг off → BrowserFetcher без пула = env-прокси, поведение байт-в-байт как раньше. newbuilding.fetch_newbuilding НЕ трогали — config/proxy_provider до неё не доходят (сигнатура без них, callers не передают), нужен отдельный рефактор.
This commit is contained in:
bot-backend
parent
e440209d91
commit
39826058ea
5 changed files with 186 additions and 6 deletions
164
tradein-mvp/backend/tests/test_kit_serp_proxy_pool.py
Normal file
164
tradein-mvp/backend/tests/test_kit_serp_proxy_pool.py
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
"""Тесты проброса browser-пула в SERP-скраперы yandex/cian (#2160).
|
||||
|
||||
Зеркало P4 (#2164) для avito: там пул раздавался только в avito-путях, а yandex/cian
|
||||
SERP создавали BrowserFetcher сами в __aenter__ без пула → весь трафик шёл через
|
||||
env-прокси браузера. Здесь проверяем, что YandexRealtyScraper / CianScraper прокидывают
|
||||
proxy_provider + use_pool=config.use_proxy_pool_browser в BrowserFetcher.
|
||||
|
||||
Инвариант ship-dark:
|
||||
- proxy_provider=None ИЛИ флаг off → BrowserFetcher(..., proxy_provider=None, use_pool=False),
|
||||
т.е. env-прокси браузера, поведение байт-в-байт как до фикса.
|
||||
- proxy_provider задан + флаг on → BrowserFetcher получает провайдер + use_pool=True.
|
||||
|
||||
BrowserFetcher монки-патчится на recording-заглушку в модуле каждого скрапера, так что
|
||||
реального HTTP к tradein-browser не происходит.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import types
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from scraper_kit.providers.cian import serp as cian_serp
|
||||
from scraper_kit.providers.yandex import serp as yandex_serp
|
||||
|
||||
|
||||
class _FakeProxyProvider:
|
||||
"""ProxyProvider-заглушка: методы no-op (в этих тестах пул не вызывается —
|
||||
BrowserFetcher замокан, acquire/release не срабатывают)."""
|
||||
|
||||
def acquire(self, provider: str) -> None:
|
||||
return None
|
||||
|
||||
def release(self, lease: object) -> None:
|
||||
return None
|
||||
|
||||
def mark_health(self, lease: object, ok: bool) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _make_recorder() -> tuple[type, list[dict[str, Any]]]:
|
||||
"""Recording-заглушка BrowserFetcher: пишет init-kwargs в общий список."""
|
||||
calls: list[dict[str, Any]] = []
|
||||
|
||||
class _RecFetcher:
|
||||
def __init__(
|
||||
self,
|
||||
source: str = "avito",
|
||||
fetch_timeout_s: float = 120.0,
|
||||
*,
|
||||
endpoint: str,
|
||||
proxy_provider: object | None = None,
|
||||
use_pool: bool = False,
|
||||
) -> None:
|
||||
calls.append(
|
||||
{
|
||||
"source": source,
|
||||
"endpoint": endpoint,
|
||||
"proxy_provider": proxy_provider,
|
||||
"use_pool": use_pool,
|
||||
}
|
||||
)
|
||||
|
||||
async def __aenter__(self) -> _RecFetcher:
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args: object) -> None:
|
||||
return None
|
||||
|
||||
return _RecFetcher, calls
|
||||
|
||||
|
||||
def _config(*, use_pool: bool) -> types.SimpleNamespace:
|
||||
return types.SimpleNamespace(
|
||||
browser_http_endpoint="http://browser:3000",
|
||||
use_proxy_pool_browser=use_pool,
|
||||
)
|
||||
|
||||
|
||||
# ── Yandex SERP ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_yandex_serp_wires_pool_when_flag_on(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""proxy_provider + флаг on → BrowserFetcher получает провайдер + use_pool=True."""
|
||||
rec_fetcher, calls = _make_recorder()
|
||||
monkeypatch.setattr(yandex_serp, "BrowserFetcher", rec_fetcher)
|
||||
provider = _FakeProxyProvider()
|
||||
|
||||
scraper = yandex_serp.YandexRealtyScraper(_config(use_pool=True), proxy_provider=provider)
|
||||
async with scraper:
|
||||
pass
|
||||
|
||||
assert len(calls) == 1
|
||||
assert calls[0]["source"] == "yandex"
|
||||
assert calls[0]["proxy_provider"] is provider
|
||||
assert calls[0]["use_pool"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_yandex_serp_no_pool_by_default(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Без provider/флага → BrowserFetcher(proxy_provider=None, use_pool=False) — ship-dark."""
|
||||
rec_fetcher, calls = _make_recorder()
|
||||
monkeypatch.setattr(yandex_serp, "BrowserFetcher", rec_fetcher)
|
||||
|
||||
scraper = yandex_serp.YandexRealtyScraper(_config(use_pool=False))
|
||||
async with scraper:
|
||||
pass
|
||||
|
||||
assert len(calls) == 1
|
||||
assert calls[0]["proxy_provider"] is None
|
||||
assert calls[0]["use_pool"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_yandex_serp_provider_without_flag_stays_dark(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""provider задан, но флаг off → use_pool=False (BrowserFetcher пул не трогает)."""
|
||||
rec_fetcher, calls = _make_recorder()
|
||||
monkeypatch.setattr(yandex_serp, "BrowserFetcher", rec_fetcher)
|
||||
provider = _FakeProxyProvider()
|
||||
|
||||
scraper = yandex_serp.YandexRealtyScraper(_config(use_pool=False), proxy_provider=provider)
|
||||
async with scraper:
|
||||
pass
|
||||
|
||||
assert calls[0]["proxy_provider"] is provider
|
||||
assert calls[0]["use_pool"] is False
|
||||
|
||||
|
||||
# ── Cian SERP ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cian_serp_wires_pool_when_flag_on(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""proxy_provider + флаг on → BrowserFetcher получает провайдер + use_pool=True."""
|
||||
rec_fetcher, calls = _make_recorder()
|
||||
monkeypatch.setattr(cian_serp, "BrowserFetcher", rec_fetcher)
|
||||
provider = _FakeProxyProvider()
|
||||
|
||||
scraper = cian_serp.CianScraper(_config(use_pool=True), proxy_provider=provider)
|
||||
async with scraper:
|
||||
pass
|
||||
|
||||
assert len(calls) == 1
|
||||
assert calls[0]["source"] == "cian"
|
||||
assert calls[0]["proxy_provider"] is provider
|
||||
assert calls[0]["use_pool"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cian_serp_no_pool_by_default(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Без provider/флага → BrowserFetcher(proxy_provider=None, use_pool=False) — ship-dark."""
|
||||
rec_fetcher, calls = _make_recorder()
|
||||
monkeypatch.setattr(cian_serp, "BrowserFetcher", rec_fetcher)
|
||||
|
||||
scraper = cian_serp.CianScraper(_config(use_pool=False))
|
||||
async with scraper:
|
||||
pass
|
||||
|
||||
assert len(calls) == 1
|
||||
assert calls[0]["proxy_provider"] is None
|
||||
assert calls[0]["use_pool"] is False
|
||||
|
|
@ -1632,6 +1632,7 @@ async def run_yandex_city_sweep(
|
|||
matcher: HouseMatcher,
|
||||
enrichment: EnrichmentJobs,
|
||||
shutdown_requested: Callable[[], bool] = lambda: False,
|
||||
proxy_provider: ProxyProvider | None = None,
|
||||
anchors: list[tuple[float, float, str]] | None = None,
|
||||
radius_m: int = 25000,
|
||||
pages_per_anchor: int = 2,
|
||||
|
|
@ -1791,7 +1792,7 @@ async def run_yandex_city_sweep(
|
|||
counters.lots_fetched,
|
||||
)
|
||||
|
||||
async with YandexRealtyScraper(config) as scraper:
|
||||
async with YandexRealtyScraper(config, proxy_provider=proxy_provider) as scraper:
|
||||
await scraper.fetch_around_multi_room(
|
||||
_a_lat,
|
||||
_a_lon,
|
||||
|
|
@ -2216,7 +2217,7 @@ async def run_cian_city_sweep(
|
|||
nonlocal anchor_lots, consecutive_failures, cian_rotations_done
|
||||
|
||||
# ── Phase 1+2: SERP + save ─────────────────────────────────
|
||||
async with CianScraper(config) as scraper:
|
||||
async with CianScraper(config, proxy_provider=proxy_provider) as scraper:
|
||||
anchor_lots = await scraper.fetch_around_multi_room(
|
||||
_a_lat,
|
||||
_a_lon,
|
||||
|
|
@ -2667,7 +2668,7 @@ async def run_cian_full_load(
|
|||
try:
|
||||
_hb_task = asyncio.create_task(_background_heartbeat())
|
||||
|
||||
async with CianScraper(config) as scraper:
|
||||
async with CianScraper(config, proxy_provider=proxy_provider) as scraper:
|
||||
scraper.request_delay_sec = request_delay_sec
|
||||
|
||||
# #1949: per-fetch hard-cancel для browser-fetch'ей внутри bucket-gather.
|
||||
|
|
@ -2865,6 +2866,7 @@ async def run_yandex_full_load(
|
|||
matcher: HouseMatcher,
|
||||
enrichment: EnrichmentJobs,
|
||||
shutdown_requested: Callable[[], bool] = lambda: False,
|
||||
proxy_provider: ProxyProvider | None = None,
|
||||
price_cap_per_bucket: int = 500,
|
||||
request_delay_sec: float = 2.0,
|
||||
concurrency: int = 4,
|
||||
|
|
|
|||
|
|
@ -520,6 +520,7 @@ async def _job_yandex_city_sweep(
|
|||
matcher=ctx.matcher,
|
||||
enrichment=ctx.enrichment,
|
||||
shutdown_requested=ctx.shutdown_requested,
|
||||
proxy_provider=ctx.proxy_provider,
|
||||
pages_per_anchor=int(params.get("pages_per_anchor", 2)),
|
||||
request_delay_sec=float(params.get("request_delay_sec", 9.0)),
|
||||
radius_m=int(params.get("radius_m", 1500)),
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ from scraper_kit.repair_state_normalizer import (
|
|||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scraper_kit.contracts import ScraperConfig
|
||||
from scraper_kit.contracts import ProxyProvider, ScraperConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -118,6 +118,7 @@ class CianScraper(BaseScraper):
|
|||
config: ScraperConfig,
|
||||
*,
|
||||
delay_provider: Callable[[str], float] | None = None,
|
||||
proxy_provider: ProxyProvider | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
# Strangler-инжекция (#2133): конфиг и провайдер задержки приходят снаружи
|
||||
|
|
@ -127,6 +128,9 @@ class CianScraper(BaseScraper):
|
|||
self._config = config
|
||||
if delay_provider is not None:
|
||||
self.request_delay_sec = delay_provider(self.name)
|
||||
# #2160: пул прокси для camoufox-браузера (за флагом use_proxy_pool_browser).
|
||||
# None → BrowserFetcher без пула = env-прокси браузера (ship-dark, как сейчас).
|
||||
self._proxy_provider = proxy_provider
|
||||
self._browser: BrowserFetcher | None = None
|
||||
|
||||
async def __aenter__(self) -> CianScraper:
|
||||
|
|
@ -136,7 +140,10 @@ class CianScraper(BaseScraper):
|
|||
# отдельный браузер+мобильный прокси для cian. Прокси ротирует IP сам, поэтому
|
||||
# code-side warm-up cookies и changeip-ротация больше не нужны.
|
||||
self._browser = BrowserFetcher(
|
||||
source="cian", endpoint=self._config.browser_http_endpoint
|
||||
source="cian",
|
||||
endpoint=self._config.browser_http_endpoint,
|
||||
proxy_provider=self._proxy_provider,
|
||||
use_pool=self._config.use_proxy_pool_browser,
|
||||
)
|
||||
await self._browser.__aenter__()
|
||||
return self
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ from scraper_kit.price_brackets import get_price_seed_brackets
|
|||
from scraper_kit.pricing import BisectionConfig, ProbeFailPolicy, ProbeResult, walk_price_range
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scraper_kit.contracts import ScraperConfig
|
||||
from scraper_kit.contracts import ProxyProvider, ScraperConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -461,6 +461,7 @@ class YandexRealtyScraper(BaseScraper):
|
|||
city: str = DEFAULT_CITY,
|
||||
*,
|
||||
delay_provider: Callable[[str], float] | None = None,
|
||||
proxy_provider: ProxyProvider | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.city = city
|
||||
|
|
@ -471,6 +472,9 @@ class YandexRealtyScraper(BaseScraper):
|
|||
self._config = config
|
||||
if delay_provider is not None:
|
||||
self.request_delay_sec = delay_provider(self.name)
|
||||
# #2160: пул прокси для camoufox-браузера (за флагом use_proxy_pool_browser).
|
||||
# None → BrowserFetcher без пула = env-прокси браузера (ship-dark, как сейчас).
|
||||
self._proxy_provider = proxy_provider
|
||||
self._browser: BrowserFetcher | None = None
|
||||
# _cffi_session retained only for _rotate_ip (changeip call).
|
||||
self._cffi_session: _CurlCffiSession | None = None
|
||||
|
|
@ -490,6 +494,8 @@ class YandexRealtyScraper(BaseScraper):
|
|||
source="yandex",
|
||||
fetch_timeout_s=_YANDEX_BROWSER_FETCH_TIMEOUT_S,
|
||||
endpoint=self._config.browser_http_endpoint,
|
||||
proxy_provider=self._proxy_provider,
|
||||
use_pool=self._config.use_proxy_pool_browser,
|
||||
)
|
||||
await self._browser.__aenter__()
|
||||
return self
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue