Merge pull request 'fix(tradein/domclick): detail-бэкфилл ходил в сайдкар мимо прокси-пула (#3197 ч.1)' (#3222) from fix/3197-domclick-backfill-proxy-pool into main

tradein-mvp/backend/app/tasks/domclick_detail_backfill.py

@@ -101,6 +101,7 @@ from app.core.shutdown import shutdown_requested
 from app.services import domclick_session as domclick_session_svc
 from app.services import scrape_runs as runs_mod
 from app.services.scrape_runs import BAN_KIND_PLATFORM, BAN_KIND_UNKNOWN
+from app.services.scraper_adapters import RealProxyProvider, RealScraperConfig
 
 logger = logging.getLogger(__name__)
 
@@ -314,9 +315,19 @@ async def run_domclick_detail_backfill(
         block_ban_kinds: Counter[str] = Counter()
 
         # Exactly ONE BrowserFetcher per run (no curl fallback for DomClick, see
-        # module docstring). source="domclick" -- infra identifier, dedicated
-        # residential proxy (scrape_proxies.provider_affinity='domclick',
-        # 173_scrape_proxies_add_domclick_affinity.sql).
+        # module docstring). source="domclick" -- infra identifier; node comes from the
+        # SHARED pool (migration 253 / #2800 dropped the dedicated-node reservation --
+        # as of 29.08 all four enabled nodes, id 1/9/10/11, have
+        # provider_affinity='any', there is no 'domclick'-affinity row anymore).
+        # proxy_provider/use_pool/environment (#3197): mandatory wiring, mirrors
+        # avito_detail_backfill.py -- without them BrowserFetcher never puts "proxy" in
+        # the /fetch body, the sidecar falls back to its own env-proxy, and the run goes
+        # around the pool entirely (no affinity selection, no scrape_proxy_source_bans,
+        # no rotation on block). environment is required too: without it the "pool
+        # empty" refusal on this path is dead code and the fetcher silently falls back
+        # to the env-proxy instead (#2616 step 1). Same defect already fixed in
+        # avito_detail_backfill (#2698) and house_imv_backfill (#2698) -- this call site
+        # was the one still missing it.
         # reuse_context=True (#3118): sidecar's browser.new_page() creates a fresh
         # isolated context on EVERY /fetch, so the once-per-run cookie injection above
         # never sees the live qrator_jsid2 the site rotates via Set-Cookie (~2.5h TTL)
@@ -325,8 +336,14 @@ async def run_domclick_detail_backfill(
         # context alive for the whole run instead, so the cookie jar evolves on its
         # own. Reset happens once per detected block (see reset_context_next below),
         # not on every subsequent fetch.
+        _cfg = RealScraperConfig()
         async with BrowserFetcher(
-            source="domclick", endpoint=settings.browser_http_endpoint, reuse_context=True
+            source="domclick",
+            endpoint=settings.browser_http_endpoint,
+            proxy_provider=RealProxyProvider(),
+            use_pool=_cfg.use_proxy_pool_browser,
+            environment=_cfg.environment,
+            reuse_context=True,
         ) as bf:
             for idx, row in enumerate(snapshot):
                 # Budget guard

tradein-mvp/backend/tests/tasks/test_domclick_detail_backfill.py

@@ -162,9 +162,14 @@ async def test_backfill_processes_snapshot_with_cookies_threaded() -> None:
     assert result.blocked == 0
     assert result.failed == 0
     assert mock_fetch.call_count == 3
-    mock_bf_cls.assert_called_once_with(
-        source="domclick", endpoint=fake_settings.browser_http_endpoint, reuse_context=True
-    )
+    # #3197: конструктор с 08.2026 несёт ещё proxy_provider/use_pool/environment
+    # (проводка прокси-пула) -- здесь проверяем только то, что нужно этому тесту,
+    # полная проводка покрыта tests/test_3197_domclick_proxy_pool_wiring.py.
+    mock_bf_cls.assert_called_once()
+    _, bf_kwargs = mock_bf_cls.call_args
+    assert bf_kwargs["source"] == "domclick"
+    assert bf_kwargs["endpoint"] == fake_settings.browser_http_endpoint
+    assert bf_kwargs["reuse_context"] is True
     for call in mock_fetch.call_args_list:
         _, kwargs = call
         assert kwargs.get("cookies") == fake_cookies

tradein-mvp/backend/tests/test_3118_domclick_warm_context.py

@@ -106,9 +106,14 @@ async def test_backfill_constructs_browser_fetcher_with_reuse_context() -> None:
             db, run_id=100, params={"batch_size": 10, "budget_sec": 3600}
         )
 
-    mock_bf_cls.assert_called_once_with(
-        source="domclick", endpoint=fake_settings.browser_http_endpoint, reuse_context=True
-    )
+    # #3197: конструктор с 08.2026 несёт ещё proxy_provider/use_pool/environment
+    # (проводка прокси-пула) -- этот тест проверяет только reuse_context, для полной
+    # проводки см. tests/test_3197_domclick_proxy_pool_wiring.py.
+    mock_bf_cls.assert_called_once()
+    _, call_kwargs = mock_bf_cls.call_args
+    assert call_kwargs["source"] == "domclick"
+    assert call_kwargs["endpoint"] == fake_settings.browser_http_endpoint
+    assert call_kwargs["reuse_context"] is True
 
 
 @pytest.mark.asyncio

tradein-mvp/backend/tests/test_3197_domclick_proxy_pool_wiring.py

@@ -0,0 +1,137 @@
+"""#3197 (часть 1, Домклик) — backfill ходил в сайдкар мимо прокси-пула.
+
+`BrowserFetcher(source="domclick", endpoint=..., reuse_context=True)` конструировался
+БЕЗ `proxy_provider`/`use_pool`/`environment` — единственные три аргумента, которые
+кладут "proxy" в тело POST /fetch (см. scraper_kit.browser_fetcher.BrowserFetcher.
+_acquire_lease). Без них сайдкар брал свой env-прокси (`SCRAPER_PROXY_URL`), и прогон
+шёл мимо пула целиком: ни выбора узла, ни `scrape_proxy_source_bans`, ни ротации при
+блоке. Тот же дефект уже чинили на соседних путях — avito_detail_backfill (#2698,
+см. test_2698_imv_sidecar_wiring.py) и house_imv_backfill (#2698) — этот call site
+оставался последним непочиненным.
+
+Сеть/БД/камуфокс замоканы; в сеть тест не ходит.
+"""
+
+from __future__ import annotations
+
+import os
+from datetime import UTC, datetime, timedelta
+from typing import Any, ClassVar
+from unittest.mock import AsyncMock, MagicMock, patch
+
+os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
+
+import pytest
+
+from app.tasks import domclick_detail_backfill as dcb
+
+_RUNS = "app.tasks.domclick_detail_backfill.runs_mod"
+_SESSION_SVC = "app.tasks.domclick_detail_backfill.domclick_session_svc"
+_SLEEP = "app.tasks.domclick_detail_backfill.asyncio.sleep"
+_FETCH = "app.tasks.domclick_detail_backfill.fetch_detail"
+_SAVE = "app.tasks.domclick_detail_backfill.save_detail_enrichment"
+
+
+class _CapturingFetcher:
+    """Зеркалит _CapturingFetcher из test_2698_imv_sidecar_wiring.py."""
+
+    captured: ClassVar[dict[str, Any]] = {}
+
+    def __init__(self, **kwargs: Any) -> None:
+        _CapturingFetcher.captured = kwargs
+
+    async def __aenter__(self) -> _CapturingFetcher:
+        return self
+
+    async def __aexit__(self, *_: object) -> None:
+        return None
+
+    def request_context_reset(self) -> None:
+        return None
+
+
+def _make_snapshot(n: int) -> list[dict]:
+    return [
+        {"id": i + 1, "source_url": f"https://ekaterinburg.domclick.ru/card/sale__flat__{i + 1}"}
+        for i in range(n)
+    ]
+
+
+def _mock_db(snapshot: list[dict]) -> MagicMock:
+    db = MagicMock()
+    sel = MagicMock()
+    sel.mappings.return_value.all.return_value = snapshot
+    db.execute.return_value = sel
+    return db
+
+
+def _mock_session_svc() -> MagicMock:
+    svc = MagicMock()
+    svc.load_session.return_value = None
+    svc.COOKIE_EXPIRY_WARN_DAYS = 5
+    svc.session_expires_at.return_value = datetime.now(tz=UTC) + timedelta(days=30)
+    return svc
+
+
+@pytest.mark.asyncio
+async def test_browser_fetcher_gets_proxy_pool_wiring(monkeypatch: pytest.MonkeyPatch) -> None:
+    """use_pool/proxy_provider/environment доезжают до BrowserFetcher из конфига."""
+    _CapturingFetcher.captured = {}
+    monkeypatch.setattr(dcb.settings, "use_proxy_pool_browser", True)
+    monkeypatch.setattr(dcb.settings, "environment", "production")
+
+    snapshot = _make_snapshot(1)
+    db = _mock_db(snapshot)
+    with (
+        patch(_SESSION_SVC, _mock_session_svc()),
+        patch(_RUNS, MagicMock()),
+        patch.object(dcb, "BrowserFetcher", _CapturingFetcher),
+        patch(_FETCH, AsyncMock(return_value=MagicMock())),
+        patch(_SAVE, return_value=True),
+        patch(_SLEEP, new_callable=AsyncMock),
+    ):
+        await dcb.run_domclick_detail_backfill(
+            db, run_id=3197, params={"batch_size": 1, "budget_sec": 3600}
+        )
+
+    captured = _CapturingFetcher.captured
+    assert captured["source"] == "domclick"
+    assert captured["proxy_provider"] is not None, "без провайдера пул не подключится"
+    assert captured["use_pool"] is True, "флаг пула должен доезжать до фетчера из конфига"
+    # #2616 шаг 1: без него отказ «пул пуст» на этом пути мёртв.
+    assert captured["environment"] == "production"
+    # #3118: reuse_context остаётся включённым — QRATOR прогрев тратится один раз
+    # на весь прогон, а не на каждую карточку.
+    assert captured["reuse_context"] is True
+
+
+@pytest.mark.asyncio
+async def test_browser_fetcher_use_pool_follows_config_off(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """use_proxy_pool_browser=False (дефолт) -> use_pool=False, provider всё равно передан."""
+    _CapturingFetcher.captured = {}
+    monkeypatch.setattr(dcb.settings, "use_proxy_pool_browser", False)
+    monkeypatch.setattr(dcb.settings, "environment", "dev")
+
+    snapshot = _make_snapshot(1)
+    db = _mock_db(snapshot)
+    with (
+        patch(_SESSION_SVC, _mock_session_svc()),
+        patch(_RUNS, MagicMock()),
+        patch.object(dcb, "BrowserFetcher", _CapturingFetcher),
+        patch(_FETCH, AsyncMock(return_value=MagicMock())),
+        patch(_SAVE, return_value=True),
+        patch(_SLEEP, new_callable=AsyncMock),
+    ):
+        await dcb.run_domclick_detail_backfill(
+            db, run_id=3197, params={"batch_size": 1, "budget_sec": 3600}
+        )
+
+    captured = _CapturingFetcher.captured
+    # proxy_provider/use_pool передаются ВМЕСТЕ (см. providers/_base.py::acquire) —
+    # при use_pool=False провайдер игнорируется, но параметр не должен пропадать из
+    # проводки: это то же самое call-site для dev и для прода.
+    assert captured["proxy_provider"] is not None
+    assert captured["use_pool"] is False
+    assert captured["environment"] == "dev"