From fd95c962bb7ce09a2d1b002c2383ac1992e8a16f Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 29 Aug 2026 15:26:14 +0300 Subject: [PATCH 1/2] =?UTF-8?q?fix(tradein/domclick):=20backfill=20=D1=85?= =?UTF-8?q?=D0=BE=D0=B4=D0=B8=D0=BB=20=D0=B2=20=D1=81=D0=B0=D0=B9=D0=B4?= =?UTF-8?q?=D0=BA=D0=B0=D1=80=20=D0=BC=D0=B8=D0=BC=D0=BE=20=D0=BF=D1=80?= =?UTF-8?q?=D0=BE=D0=BA=D1=81=D0=B8-=D0=BF=D1=83=D0=BB=D0=B0=20(#3197=20?= =?UTF-8?q?=D1=87.1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BrowserFetcher(source="domclick", reuse_context=True) конструировался без proxy_provider/use_pool/environment -- тела POST /fetch не несли "proxy", сайдкар брал свой env-прокси, и прогон шёл мимо пула целиком: ни выбора узла по affinity, ни scrape_proxy_source_bans, ни ротации при блоке. Тот же дефект уже чинили на avito_detail_backfill/house_imv_backfill (#2698) -- этот call site оставался последним непочиненным. environment обязателен: без него отказ «пул пуст» на этом пути мёртв (#2616 шаг 1). reuse_context=True сохранён без изменений. Заодно поправлен устаревший комментарий над конструктором: ссылался на scrape_proxies.provider_affinity='domclick' и миграцию 173 -- на проде такого больше нет (миграция 253 сняла резервацию узла, #2800), все четыре включённых узла (id 1/9/10/11) имеют provider_affinity='any'. test_3118_domclick_warm_context.py обновлён под новую сигнатуру вызова (assert_called_once_with -> точечная проверка нужных kwargs). --- .../app/tasks/domclick_detail_backfill.py | 25 +++- .../tests/test_3118_domclick_warm_context.py | 11 +- .../test_3197_domclick_proxy_pool_wiring.py | 137 ++++++++++++++++++ 3 files changed, 166 insertions(+), 7 deletions(-) create mode 100644 tradein-mvp/backend/tests/test_3197_domclick_proxy_pool_wiring.py diff --git a/tradein-mvp/backend/app/tasks/domclick_detail_backfill.py b/tradein-mvp/backend/app/tasks/domclick_detail_backfill.py index 23804881..da802ffe 100644 --- a/tradein-mvp/backend/app/tasks/domclick_detail_backfill.py +++ b/tradein-mvp/backend/app/tasks/domclick_detail_backfill.py @@ -101,6 +101,7 @@ from app.core.shutdown import shutdown_requested from app.services import domclick_session as domclick_session_svc from app.services import scrape_runs as runs_mod from app.services.scrape_runs import BAN_KIND_PLATFORM, BAN_KIND_UNKNOWN +from app.services.scraper_adapters import RealProxyProvider, RealScraperConfig logger = logging.getLogger(__name__) @@ -314,9 +315,19 @@ async def run_domclick_detail_backfill( block_ban_kinds: Counter[str] = Counter() # Exactly ONE BrowserFetcher per run (no curl fallback for DomClick, see - # module docstring). source="domclick" -- infra identifier, dedicated - # residential proxy (scrape_proxies.provider_affinity='domclick', - # 173_scrape_proxies_add_domclick_affinity.sql). + # module docstring). source="domclick" -- infra identifier; node comes from the + # SHARED pool (migration 253 / #2800 dropped the dedicated-node reservation -- + # as of 29.08 all four enabled nodes, id 1/9/10/11, have + # provider_affinity='any', there is no 'domclick'-affinity row anymore). + # proxy_provider/use_pool/environment (#3197): mandatory wiring, mirrors + # avito_detail_backfill.py -- without them BrowserFetcher never puts "proxy" in + # the /fetch body, the sidecar falls back to its own env-proxy, and the run goes + # around the pool entirely (no affinity selection, no scrape_proxy_source_bans, + # no rotation on block). environment is required too: without it the "pool + # empty" refusal on this path is dead code and the fetcher silently falls back + # to the env-proxy instead (#2616 step 1). Same defect already fixed in + # avito_detail_backfill (#2698) and house_imv_backfill (#2698) -- this call site + # was the one still missing it. # reuse_context=True (#3118): sidecar's browser.new_page() creates a fresh # isolated context on EVERY /fetch, so the once-per-run cookie injection above # never sees the live qrator_jsid2 the site rotates via Set-Cookie (~2.5h TTL) @@ -325,8 +336,14 @@ async def run_domclick_detail_backfill( # context alive for the whole run instead, so the cookie jar evolves on its # own. Reset happens once per detected block (see reset_context_next below), # not on every subsequent fetch. + _cfg = RealScraperConfig() async with BrowserFetcher( - source="domclick", endpoint=settings.browser_http_endpoint, reuse_context=True + source="domclick", + endpoint=settings.browser_http_endpoint, + proxy_provider=RealProxyProvider(), + use_pool=_cfg.use_proxy_pool_browser, + environment=_cfg.environment, + reuse_context=True, ) as bf: for idx, row in enumerate(snapshot): # Budget guard diff --git a/tradein-mvp/backend/tests/test_3118_domclick_warm_context.py b/tradein-mvp/backend/tests/test_3118_domclick_warm_context.py index 5827345a..392d7f93 100644 --- a/tradein-mvp/backend/tests/test_3118_domclick_warm_context.py +++ b/tradein-mvp/backend/tests/test_3118_domclick_warm_context.py @@ -106,9 +106,14 @@ async def test_backfill_constructs_browser_fetcher_with_reuse_context() -> None: db, run_id=100, params={"batch_size": 10, "budget_sec": 3600} ) - mock_bf_cls.assert_called_once_with( - source="domclick", endpoint=fake_settings.browser_http_endpoint, reuse_context=True - ) + # #3197: конструктор с 08.2026 несёт ещё proxy_provider/use_pool/environment + # (проводка прокси-пула) -- этот тест проверяет только reuse_context, для полной + # проводки см. tests/test_3197_domclick_proxy_pool_wiring.py. + mock_bf_cls.assert_called_once() + _, call_kwargs = mock_bf_cls.call_args + assert call_kwargs["source"] == "domclick" + assert call_kwargs["endpoint"] == fake_settings.browser_http_endpoint + assert call_kwargs["reuse_context"] is True @pytest.mark.asyncio diff --git a/tradein-mvp/backend/tests/test_3197_domclick_proxy_pool_wiring.py b/tradein-mvp/backend/tests/test_3197_domclick_proxy_pool_wiring.py new file mode 100644 index 00000000..94e86b4c --- /dev/null +++ b/tradein-mvp/backend/tests/test_3197_domclick_proxy_pool_wiring.py @@ -0,0 +1,137 @@ +"""#3197 (часть 1, Домклик) — backfill ходил в сайдкар мимо прокси-пула. + +`BrowserFetcher(source="domclick", endpoint=..., reuse_context=True)` конструировался +БЕЗ `proxy_provider`/`use_pool`/`environment` — единственные три аргумента, которые +кладут "proxy" в тело POST /fetch (см. scraper_kit.browser_fetcher.BrowserFetcher. +_acquire_lease). Без них сайдкар брал свой env-прокси (`SCRAPER_PROXY_URL`), и прогон +шёл мимо пула целиком: ни выбора узла, ни `scrape_proxy_source_bans`, ни ротации при +блоке. Тот же дефект уже чинили на соседних путях — avito_detail_backfill (#2698, +см. test_2698_imv_sidecar_wiring.py) и house_imv_backfill (#2698) — этот call site +оставался последним непочиненным. + +Сеть/БД/камуфокс замоканы; в сеть тест не ходит. +""" + +from __future__ import annotations + +import os +from datetime import UTC, datetime, timedelta +from typing import Any, ClassVar +from unittest.mock import AsyncMock, MagicMock, patch + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + +import pytest + +from app.tasks import domclick_detail_backfill as dcb + +_RUNS = "app.tasks.domclick_detail_backfill.runs_mod" +_SESSION_SVC = "app.tasks.domclick_detail_backfill.domclick_session_svc" +_SLEEP = "app.tasks.domclick_detail_backfill.asyncio.sleep" +_FETCH = "app.tasks.domclick_detail_backfill.fetch_detail" +_SAVE = "app.tasks.domclick_detail_backfill.save_detail_enrichment" + + +class _CapturingFetcher: + """Зеркалит _CapturingFetcher из test_2698_imv_sidecar_wiring.py.""" + + captured: ClassVar[dict[str, Any]] = {} + + def __init__(self, **kwargs: Any) -> None: + _CapturingFetcher.captured = kwargs + + async def __aenter__(self) -> _CapturingFetcher: + return self + + async def __aexit__(self, *_: object) -> None: + return None + + def request_context_reset(self) -> None: + return None + + +def _make_snapshot(n: int) -> list[dict]: + return [ + {"id": i + 1, "source_url": f"https://ekaterinburg.domclick.ru/card/sale__flat__{i + 1}"} + for i in range(n) + ] + + +def _mock_db(snapshot: list[dict]) -> MagicMock: + db = MagicMock() + sel = MagicMock() + sel.mappings.return_value.all.return_value = snapshot + db.execute.return_value = sel + return db + + +def _mock_session_svc() -> MagicMock: + svc = MagicMock() + svc.load_session.return_value = None + svc.COOKIE_EXPIRY_WARN_DAYS = 5 + svc.session_expires_at.return_value = datetime.now(tz=UTC) + timedelta(days=30) + return svc + + +@pytest.mark.asyncio +async def test_browser_fetcher_gets_proxy_pool_wiring(monkeypatch: pytest.MonkeyPatch) -> None: + """use_pool/proxy_provider/environment доезжают до BrowserFetcher из конфига.""" + _CapturingFetcher.captured = {} + monkeypatch.setattr(dcb.settings, "use_proxy_pool_browser", True) + monkeypatch.setattr(dcb.settings, "environment", "production") + + snapshot = _make_snapshot(1) + db = _mock_db(snapshot) + with ( + patch(_SESSION_SVC, _mock_session_svc()), + patch(_RUNS, MagicMock()), + patch.object(dcb, "BrowserFetcher", _CapturingFetcher), + patch(_FETCH, AsyncMock(return_value=MagicMock())), + patch(_SAVE, return_value=True), + patch(_SLEEP, new_callable=AsyncMock), + ): + await dcb.run_domclick_detail_backfill( + db, run_id=3197, params={"batch_size": 1, "budget_sec": 3600} + ) + + captured = _CapturingFetcher.captured + assert captured["source"] == "domclick" + assert captured["proxy_provider"] is not None, "без провайдера пул не подключится" + assert captured["use_pool"] is True, "флаг пула должен доезжать до фетчера из конфига" + # #2616 шаг 1: без него отказ «пул пуст» на этом пути мёртв. + assert captured["environment"] == "production" + # #3118: reuse_context остаётся включённым — QRATOR прогрев тратится один раз + # на весь прогон, а не на каждую карточку. + assert captured["reuse_context"] is True + + +@pytest.mark.asyncio +async def test_browser_fetcher_use_pool_follows_config_off( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """use_proxy_pool_browser=False (дефолт) -> use_pool=False, provider всё равно передан.""" + _CapturingFetcher.captured = {} + monkeypatch.setattr(dcb.settings, "use_proxy_pool_browser", False) + monkeypatch.setattr(dcb.settings, "environment", "dev") + + snapshot = _make_snapshot(1) + db = _mock_db(snapshot) + with ( + patch(_SESSION_SVC, _mock_session_svc()), + patch(_RUNS, MagicMock()), + patch.object(dcb, "BrowserFetcher", _CapturingFetcher), + patch(_FETCH, AsyncMock(return_value=MagicMock())), + patch(_SAVE, return_value=True), + patch(_SLEEP, new_callable=AsyncMock), + ): + await dcb.run_domclick_detail_backfill( + db, run_id=3197, params={"batch_size": 1, "budget_sec": 3600} + ) + + captured = _CapturingFetcher.captured + # proxy_provider/use_pool передаются ВМЕСТЕ (см. providers/_base.py::acquire) — + # при use_pool=False провайдер игнорируется, но параметр не должен пропадать из + # проводки: это то же самое call-site для dev и для прода. + assert captured["proxy_provider"] is not None + assert captured["use_pool"] is False + assert captured["environment"] == "dev" -- 2.45.3 From 301d803ac50f4da327e36ef0b4c15c01c750a0ed Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 29 Aug 2026 15:40:56 +0300 Subject: [PATCH 2/2] =?UTF-8?q?test(tradein/domclick):=20=D0=B2=D1=82?= =?UTF-8?q?=D0=BE=D1=80=D0=BE=D0=B9=20assert=5Fcalled=5Fonce=5Fwith=20?= =?UTF-8?q?=D0=BD=D0=B0=20=D0=BA=D0=BE=D0=BD=D1=81=D1=82=D1=80=D1=83=D0=BA?= =?UTF-8?q?=D1=82=D0=BE=D1=80=D0=B5=20(#3197=20=D1=87.1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI Trade-In / backend-tests упал на tests/tasks/test_domclick_detail_backfill.py:165 — там тот же хрупкий assert_called_once_with, что уже был поправлен в tests/test_3118_domclick_warm_context.py: он фиксирует ТОЧНУЮ сигнатуру вызова BrowserFetcher и ломается на любом новом kwarg. Лечение то же самое: assert_called_once() + точечная проверка source/endpoint/ reuse_context. Полная проводка пула покрыта отдельным tests/test_3197_domclick_proxy_pool_wiring.py. Причина пропуска: локально прогонялась выборка из трёх файлов, а не весь набор. Теперь прогнан весь: 4974 passed, 37 skipped, 0 failed. --- .../tests/tasks/test_domclick_detail_backfill.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tradein-mvp/backend/tests/tasks/test_domclick_detail_backfill.py b/tradein-mvp/backend/tests/tasks/test_domclick_detail_backfill.py index ff10c056..dc68ad7c 100644 --- a/tradein-mvp/backend/tests/tasks/test_domclick_detail_backfill.py +++ b/tradein-mvp/backend/tests/tasks/test_domclick_detail_backfill.py @@ -162,9 +162,14 @@ async def test_backfill_processes_snapshot_with_cookies_threaded() -> None: assert result.blocked == 0 assert result.failed == 0 assert mock_fetch.call_count == 3 - mock_bf_cls.assert_called_once_with( - source="domclick", endpoint=fake_settings.browser_http_endpoint, reuse_context=True - ) + # #3197: конструктор с 08.2026 несёт ещё proxy_provider/use_pool/environment + # (проводка прокси-пула) -- здесь проверяем только то, что нужно этому тесту, + # полная проводка покрыта tests/test_3197_domclick_proxy_pool_wiring.py. + mock_bf_cls.assert_called_once() + _, bf_kwargs = mock_bf_cls.call_args + assert bf_kwargs["source"] == "domclick" + assert bf_kwargs["endpoint"] == fake_settings.browser_http_endpoint + assert bf_kwargs["reuse_context"] is True for call in mock_fetch.call_args_list: _, kwargs = call assert kwargs.get("cookies") == fake_cookies -- 2.45.3