diff --git a/tradein-mvp/backend/app/tasks/domclick_detail_backfill.py b/tradein-mvp/backend/app/tasks/domclick_detail_backfill.py index 23804881..da802ffe 100644 --- a/tradein-mvp/backend/app/tasks/domclick_detail_backfill.py +++ b/tradein-mvp/backend/app/tasks/domclick_detail_backfill.py @@ -101,6 +101,7 @@ from app.core.shutdown import shutdown_requested from app.services import domclick_session as domclick_session_svc from app.services import scrape_runs as runs_mod from app.services.scrape_runs import BAN_KIND_PLATFORM, BAN_KIND_UNKNOWN +from app.services.scraper_adapters import RealProxyProvider, RealScraperConfig logger = logging.getLogger(__name__) @@ -314,9 +315,19 @@ async def run_domclick_detail_backfill( block_ban_kinds: Counter[str] = Counter() # Exactly ONE BrowserFetcher per run (no curl fallback for DomClick, see - # module docstring). source="domclick" -- infra identifier, dedicated - # residential proxy (scrape_proxies.provider_affinity='domclick', - # 173_scrape_proxies_add_domclick_affinity.sql). + # module docstring). source="domclick" -- infra identifier; node comes from the + # SHARED pool (migration 253 / #2800 dropped the dedicated-node reservation -- + # as of 29.08 all four enabled nodes, id 1/9/10/11, have + # provider_affinity='any', there is no 'domclick'-affinity row anymore). + # proxy_provider/use_pool/environment (#3197): mandatory wiring, mirrors + # avito_detail_backfill.py -- without them BrowserFetcher never puts "proxy" in + # the /fetch body, the sidecar falls back to its own env-proxy, and the run goes + # around the pool entirely (no affinity selection, no scrape_proxy_source_bans, + # no rotation on block). environment is required too: without it the "pool + # empty" refusal on this path is dead code and the fetcher silently falls back + # to the env-proxy instead (#2616 step 1). Same defect already fixed in + # avito_detail_backfill (#2698) and house_imv_backfill (#2698) -- this call site + # was the one still missing it. # reuse_context=True (#3118): sidecar's browser.new_page() creates a fresh # isolated context on EVERY /fetch, so the once-per-run cookie injection above # never sees the live qrator_jsid2 the site rotates via Set-Cookie (~2.5h TTL) @@ -325,8 +336,14 @@ async def run_domclick_detail_backfill( # context alive for the whole run instead, so the cookie jar evolves on its # own. Reset happens once per detected block (see reset_context_next below), # not on every subsequent fetch. + _cfg = RealScraperConfig() async with BrowserFetcher( - source="domclick", endpoint=settings.browser_http_endpoint, reuse_context=True + source="domclick", + endpoint=settings.browser_http_endpoint, + proxy_provider=RealProxyProvider(), + use_pool=_cfg.use_proxy_pool_browser, + environment=_cfg.environment, + reuse_context=True, ) as bf: for idx, row in enumerate(snapshot): # Budget guard diff --git a/tradein-mvp/backend/tests/test_3118_domclick_warm_context.py b/tradein-mvp/backend/tests/test_3118_domclick_warm_context.py index 5827345a..392d7f93 100644 --- a/tradein-mvp/backend/tests/test_3118_domclick_warm_context.py +++ b/tradein-mvp/backend/tests/test_3118_domclick_warm_context.py @@ -106,9 +106,14 @@ async def test_backfill_constructs_browser_fetcher_with_reuse_context() -> None: db, run_id=100, params={"batch_size": 10, "budget_sec": 3600} ) - mock_bf_cls.assert_called_once_with( - source="domclick", endpoint=fake_settings.browser_http_endpoint, reuse_context=True - ) + # #3197: конструктор с 08.2026 несёт ещё proxy_provider/use_pool/environment + # (проводка прокси-пула) -- этот тест проверяет только reuse_context, для полной + # проводки см. tests/test_3197_domclick_proxy_pool_wiring.py. + mock_bf_cls.assert_called_once() + _, call_kwargs = mock_bf_cls.call_args + assert call_kwargs["source"] == "domclick" + assert call_kwargs["endpoint"] == fake_settings.browser_http_endpoint + assert call_kwargs["reuse_context"] is True @pytest.mark.asyncio diff --git a/tradein-mvp/backend/tests/test_3197_domclick_proxy_pool_wiring.py b/tradein-mvp/backend/tests/test_3197_domclick_proxy_pool_wiring.py new file mode 100644 index 00000000..94e86b4c --- /dev/null +++ b/tradein-mvp/backend/tests/test_3197_domclick_proxy_pool_wiring.py @@ -0,0 +1,137 @@ +"""#3197 (часть 1, Домклик) — backfill ходил в сайдкар мимо прокси-пула. + +`BrowserFetcher(source="domclick", endpoint=..., reuse_context=True)` конструировался +БЕЗ `proxy_provider`/`use_pool`/`environment` — единственные три аргумента, которые +кладут "proxy" в тело POST /fetch (см. scraper_kit.browser_fetcher.BrowserFetcher. +_acquire_lease). Без них сайдкар брал свой env-прокси (`SCRAPER_PROXY_URL`), и прогон +шёл мимо пула целиком: ни выбора узла, ни `scrape_proxy_source_bans`, ни ротации при +блоке. Тот же дефект уже чинили на соседних путях — avito_detail_backfill (#2698, +см. test_2698_imv_sidecar_wiring.py) и house_imv_backfill (#2698) — этот call site +оставался последним непочиненным. + +Сеть/БД/камуфокс замоканы; в сеть тест не ходит. +""" + +from __future__ import annotations + +import os +from datetime import UTC, datetime, timedelta +from typing import Any, ClassVar +from unittest.mock import AsyncMock, MagicMock, patch + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + +import pytest + +from app.tasks import domclick_detail_backfill as dcb + +_RUNS = "app.tasks.domclick_detail_backfill.runs_mod" +_SESSION_SVC = "app.tasks.domclick_detail_backfill.domclick_session_svc" +_SLEEP = "app.tasks.domclick_detail_backfill.asyncio.sleep" +_FETCH = "app.tasks.domclick_detail_backfill.fetch_detail" +_SAVE = "app.tasks.domclick_detail_backfill.save_detail_enrichment" + + +class _CapturingFetcher: + """Зеркалит _CapturingFetcher из test_2698_imv_sidecar_wiring.py.""" + + captured: ClassVar[dict[str, Any]] = {} + + def __init__(self, **kwargs: Any) -> None: + _CapturingFetcher.captured = kwargs + + async def __aenter__(self) -> _CapturingFetcher: + return self + + async def __aexit__(self, *_: object) -> None: + return None + + def request_context_reset(self) -> None: + return None + + +def _make_snapshot(n: int) -> list[dict]: + return [ + {"id": i + 1, "source_url": f"https://ekaterinburg.domclick.ru/card/sale__flat__{i + 1}"} + for i in range(n) + ] + + +def _mock_db(snapshot: list[dict]) -> MagicMock: + db = MagicMock() + sel = MagicMock() + sel.mappings.return_value.all.return_value = snapshot + db.execute.return_value = sel + return db + + +def _mock_session_svc() -> MagicMock: + svc = MagicMock() + svc.load_session.return_value = None + svc.COOKIE_EXPIRY_WARN_DAYS = 5 + svc.session_expires_at.return_value = datetime.now(tz=UTC) + timedelta(days=30) + return svc + + +@pytest.mark.asyncio +async def test_browser_fetcher_gets_proxy_pool_wiring(monkeypatch: pytest.MonkeyPatch) -> None: + """use_pool/proxy_provider/environment доезжают до BrowserFetcher из конфига.""" + _CapturingFetcher.captured = {} + monkeypatch.setattr(dcb.settings, "use_proxy_pool_browser", True) + monkeypatch.setattr(dcb.settings, "environment", "production") + + snapshot = _make_snapshot(1) + db = _mock_db(snapshot) + with ( + patch(_SESSION_SVC, _mock_session_svc()), + patch(_RUNS, MagicMock()), + patch.object(dcb, "BrowserFetcher", _CapturingFetcher), + patch(_FETCH, AsyncMock(return_value=MagicMock())), + patch(_SAVE, return_value=True), + patch(_SLEEP, new_callable=AsyncMock), + ): + await dcb.run_domclick_detail_backfill( + db, run_id=3197, params={"batch_size": 1, "budget_sec": 3600} + ) + + captured = _CapturingFetcher.captured + assert captured["source"] == "domclick" + assert captured["proxy_provider"] is not None, "без провайдера пул не подключится" + assert captured["use_pool"] is True, "флаг пула должен доезжать до фетчера из конфига" + # #2616 шаг 1: без него отказ «пул пуст» на этом пути мёртв. + assert captured["environment"] == "production" + # #3118: reuse_context остаётся включённым — QRATOR прогрев тратится один раз + # на весь прогон, а не на каждую карточку. + assert captured["reuse_context"] is True + + +@pytest.mark.asyncio +async def test_browser_fetcher_use_pool_follows_config_off( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """use_proxy_pool_browser=False (дефолт) -> use_pool=False, provider всё равно передан.""" + _CapturingFetcher.captured = {} + monkeypatch.setattr(dcb.settings, "use_proxy_pool_browser", False) + monkeypatch.setattr(dcb.settings, "environment", "dev") + + snapshot = _make_snapshot(1) + db = _mock_db(snapshot) + with ( + patch(_SESSION_SVC, _mock_session_svc()), + patch(_RUNS, MagicMock()), + patch.object(dcb, "BrowserFetcher", _CapturingFetcher), + patch(_FETCH, AsyncMock(return_value=MagicMock())), + patch(_SAVE, return_value=True), + patch(_SLEEP, new_callable=AsyncMock), + ): + await dcb.run_domclick_detail_backfill( + db, run_id=3197, params={"batch_size": 1, "budget_sec": 3600} + ) + + captured = _CapturingFetcher.captured + # proxy_provider/use_pool передаются ВМЕСТЕ (см. providers/_base.py::acquire) — + # при use_pool=False провайдер игнорируется, но параметр не должен пропадать из + # проводки: это то же самое call-site для dev и для прода. + assert captured["proxy_provider"] is not None + assert captured["use_pool"] is False + assert captured["environment"] == "dev"