From cc3ff7c3d2ca9f1d40be5b81329688756e237aeb Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 20 Jun 2026 15:55:24 +0300 Subject: [PATCH] =?UTF-8?q?feat(tradein/scrapers):=20detail=5Fbackfill=20?= =?UTF-8?q?=D1=87=D0=B5=D1=80=D0=B5=D0=B7=20curl=5Fcffi+backconnect=20?= =?UTF-8?q?=D0=B2=D0=BC=D0=B5=D1=81=D1=82=D0=BE=20=D0=B1=D1=80=D0=B0=D1=83?= =?UTF-8?q?=D0=B7=D0=B5=D1=80=D0=B0/auv=20(=D1=80=D0=B0=D0=B7=D0=B2=D1=8F?= =?UTF-8?q?=D0=B7=D0=BA=D0=B0=20=D0=BF=D1=80=D0=BE=D0=BA=D1=81=D0=B8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SERP (full_load/city_sweep) и detail_backfill делили один прокси-аккаунт auv (~5 коннектов). Browser-фетч в backfill открывал десятки CONNECT-туннелей → cap превышался → HTTP 500/краши. - config.py: avito_detail_backfill_use_curl=True (ENV AVITO_DETAIL_BACKFILL_USE_CURL) - avito_detail_backfill.py: browser_mode гейтится флагом; use_curl=True → session=None, browser_fetcher=None → fetch_detail строит _build_detail_session() через scraper_proxy_url (backconnect mproxy, авто-ротация, 1 коннект/запрос) - tests: 2 новых теста (use_curl=True → BrowserFetcher не создаётся; False → создаётся) --- tradein-mvp/backend/app/core/config.py | 12 +++ .../app/tasks/avito_detail_backfill.py | 27 ++++-- .../tests/tasks/test_avito_detail_backfill.py | 85 +++++++++++++++++++ 3 files changed, 119 insertions(+), 5 deletions(-) diff --git a/tradein-mvp/backend/app/core/config.py b/tradein-mvp/backend/app/core/config.py index 05366fd1..4916478e 100644 --- a/tradein-mvp/backend/app/core/config.py +++ b/tradein-mvp/backend/app/core/config.py @@ -346,6 +346,18 @@ class Settings(BaseSettings): cian_login_success_cookie: str = "DMIR_AUTH" cian_login_wait_ms: int = 4000 + # detail_backfill через curl_cffi+backconnect (mproxy) вместо браузера/auv. + # SERP (full_load/city_sweep) и detail_backfill делят один прокси-аккаунт auv + # (~5 параллельных коннектов); browser-фетч в backfill открывает десятки коннектов + # → cap превышается → HTTP 500 / краши. Backconnect (mproxy, авто-ротация, + # 1 коннект/запрос) развязывает прокси-аккаунты. + # True (дефолт) = curl_cffi через settings.scraper_proxy_url (backconnect mproxy). + # False = старое browser-поведение (BrowserFetcher/auv, как scraper_fetch_mode). + # ENV: AVITO_DETAIL_BACKFILL_USE_CURL. + avito_detail_backfill_use_curl: bool = Field( + default=True, validation_alias="AVITO_DETAIL_BACKFILL_USE_CURL" + ) + # ── #884/#905/#1805: BrowserFetcher — HTTP-клиент к tradein-browser ───────── # scraper_fetch_mode: "browser" (дефолт с #1805 — HTTP POST к tradein-browser # /fetch через per-provider camoufox + ротирующий backconnect-прокси) или diff --git a/tradein-mvp/backend/app/tasks/avito_detail_backfill.py b/tradein-mvp/backend/app/tasks/avito_detail_backfill.py index 2ed2648f..e4ebb81c 100644 --- a/tradein-mvp/backend/app/tasks/avito_detail_backfill.py +++ b/tradein-mvp/backend/app/tasks/avito_detail_backfill.py @@ -89,7 +89,11 @@ async def run_avito_detail_backfill( counters = AvitoDetailBackfillResult() current_counters: dict[str, int] = counters.to_dict() - browser_mode = settings.scraper_fetch_mode == "browser" + # Режим фетча: curl_cffi+backconnect (mproxy) когда use_curl=True; иначе браузер/auv. + # avito_detail_backfill_use_curl=True переопределяет scraper_fetch_mode — detail_backfill + # всегда идёт через backconnect чтобы не конкурировать за auv с SERP (city_sweep/full_load). + use_curl = settings.avito_detail_backfill_use_curl + browser_mode = settings.scraper_fetch_mode == "browser" and not use_curl session: AsyncSession | None = None browser_fetcher: BrowserFetcher | None = None own_session = False @@ -98,14 +102,25 @@ async def run_avito_detail_backfill( scraper = AvitoScraper() start = time.monotonic() + logger.info( + "avito_detail_backfill: run_id=%d mode=%s", + run_id, + "curl/backconnect" if use_curl else "browser", + ) + try: - # Setup session (mirrors run_avito_pipeline lines 161-183) + # Setup session (mirrors run_avito_pipeline lines 161-183). + # use_curl=True: session=None + browser_fetcher=None → fetch_detail строит + # эфемерную _build_detail_session() через settings.scraper_proxy_url (backconnect). + # use_curl=False (legacy): browser_mode=True → BrowserFetcher/auv, как раньше. if browser_mode: browser_fetcher = BrowserFetcher(source="avito") await browser_fetcher.__aenter__() own_browser = True scraper._browser = browser_fetcher - else: + elif not use_curl: + # curl_cffi legacy path (scraper_fetch_mode="curl_cffi", use_curl=False): + # строим shared сессию через auv, как делал scrape_pipeline. own_session = True session = AsyncSession( impersonate="chrome120", @@ -114,6 +129,8 @@ async def run_avito_detail_backfill( proxies=_avito_proxies(), ) scraper._cffi = session + # use_curl=True: ничего не строим — fetch_detail вызывает _build_detail_session() + # (scraper_proxy_url = backconnect mproxy) на каждый запрос. runs_mod.update_heartbeat(db, run_id, current_counters) @@ -156,13 +173,13 @@ async def run_avito_detail_backfill( logger.info( "avito_detail_backfill: run_id=%d snapshot=%d (budget=%.0fs " - "delay=%.1fs max_blocks=%d browser=%s)", + "delay=%.1fs max_blocks=%d mode=%s)", run_id, len(snapshot), budget_sec, request_delay_sec, max_consecutive_blocks, - browser_mode, + "curl/backconnect" if use_curl else "browser", ) consecutive_blocks = 0 diff --git a/tradein-mvp/backend/tests/tasks/test_avito_detail_backfill.py b/tradein-mvp/backend/tests/tasks/test_avito_detail_backfill.py index 93edf1ca..fd4fcd05 100644 --- a/tradein-mvp/backend/tests/tasks/test_avito_detail_backfill.py +++ b/tradein-mvp/backend/tests/tasks/test_avito_detail_backfill.py @@ -270,3 +270,88 @@ async def test_backfill_fetch_exception_continues() -> None: assert result.attempted == 2 db.rollback.assert_called() runs.mark_done.assert_called_once() + + +_BROWSER_FETCHER = "app.tasks.avito_detail_backfill.BrowserFetcher" + + +@pytest.mark.asyncio +async def test_backfill_use_curl_flag_skips_browser_fetcher() -> None: + """avito_detail_backfill_use_curl=True: BrowserFetcher не создаётся, + fetch_detail вызывается с browser_fetcher=None (curl/backconnect путь). + """ + snapshot = _make_snapshot(1) + db = _mock_db(snapshot) + runs = MagicMock() + mock_enrichment = MagicMock() + mock_fetch = AsyncMock(return_value=mock_enrichment) + mock_save = MagicMock(return_value=True) + # Флаг use_curl=True, fetch_mode=browser (но флаг перекрывает) + fake_settings = MagicMock( + scraper_fetch_mode="browser", + avito_detail_backfill_use_curl=True, + ) + with ( + patch(_SETTINGS, fake_settings), + patch(_SESSION), + patch(_SCRAPER), + patch(_BROWSER_FETCHER) as mock_bf_cls, + patch(_RUNS, runs), + patch(_FETCH, mock_fetch), + patch(_SAVE, mock_save), + patch(_SLEEP, new_callable=AsyncMock), + ): + result = await run_avito_detail_backfill( + db, run_id=9, params={"batch_size": 10, "budget_sec": 3600} + ) + + # BrowserFetcher не должен быть создан + mock_bf_cls.assert_not_called() + # fetch_detail вызван с browser_fetcher=None (curl путь) + assert mock_fetch.call_count == 1 + _, kwargs = mock_fetch.call_args + assert kwargs.get("browser_fetcher") is None + assert result.enriched == 1 + runs.mark_done.assert_called_once() + + +@pytest.mark.asyncio +async def test_backfill_use_curl_false_creates_browser_fetcher() -> None: + """avito_detail_backfill_use_curl=False + scraper_fetch_mode='browser': + BrowserFetcher создаётся (legacy browser/auv путь). + """ + snapshot = _make_snapshot(1) + db = _mock_db(snapshot) + runs = MagicMock() + mock_enrichment = MagicMock() + mock_fetch = AsyncMock(return_value=mock_enrichment) + mock_save = MagicMock(return_value=True) + fake_settings = MagicMock( + scraper_fetch_mode="browser", + avito_detail_backfill_use_curl=False, + ) + mock_bf_instance = AsyncMock() + mock_bf_instance.__aenter__ = AsyncMock(return_value=mock_bf_instance) + mock_bf_instance.__aexit__ = AsyncMock(return_value=False) + with ( + patch(_SETTINGS, fake_settings), + patch(_SESSION), + patch(_SCRAPER), + patch(_BROWSER_FETCHER, return_value=mock_bf_instance) as mock_bf_cls, + patch(_RUNS, runs), + patch(_FETCH, mock_fetch), + patch(_SAVE, mock_save), + patch(_SLEEP, new_callable=AsyncMock), + ): + result = await run_avito_detail_backfill( + db, run_id=10, params={"batch_size": 10, "budget_sec": 3600} + ) + + # BrowserFetcher должен быть создан (source="avito") + mock_bf_cls.assert_called_once_with(source="avito") + # fetch_detail вызван с browser_fetcher установленным (не None) + assert mock_fetch.call_count == 1 + _, kwargs = mock_fetch.call_args + assert kwargs.get("browser_fetcher") is not None + assert result.enriched == 1 + runs.mark_done.assert_called_once() -- 2.45.3