Merge pull request 'feat(tradein/scrapers): detail_backfill через curl_cffi+backconnect вместо браузера/auv (развязка прокси)' (#1838) from feat/tradein-detail-backfill-curl into main

Reviewed-on: #1838

tradein-mvp/backend/app/core/config.py

@@ -346,6 +346,18 @@ class Settings(BaseSettings):
     cian_login_success_cookie: str = "DMIR_AUTH"
     cian_login_wait_ms: int = 4000
 
+    # detail_backfill через curl_cffi+backconnect (mproxy) вместо браузера/auv.
+    # SERP (full_load/city_sweep) и detail_backfill делят один прокси-аккаунт auv
+    # (~5 параллельных коннектов); browser-фетч в backfill открывает десятки коннектов
+    # → cap превышается → HTTP 500 / краши. Backconnect (mproxy, авто-ротация,
+    # 1 коннект/запрос) развязывает прокси-аккаунты.
+    # True (дефолт) = curl_cffi через settings.scraper_proxy_url (backconnect mproxy).
+    # False = старое browser-поведение (BrowserFetcher/auv, как scraper_fetch_mode).
+    # ENV: AVITO_DETAIL_BACKFILL_USE_CURL.
+    avito_detail_backfill_use_curl: bool = Field(
+        default=True, validation_alias="AVITO_DETAIL_BACKFILL_USE_CURL"
+    )
+
     # ── #884/#905/#1805: BrowserFetcher — HTTP-клиент к tradein-browser ─────────
     # scraper_fetch_mode: "browser" (дефолт с #1805 — HTTP POST к tradein-browser
     # /fetch через per-provider camoufox + ротирующий backconnect-прокси) или

tradein-mvp/backend/app/tasks/avito_detail_backfill.py

@@ -89,7 +89,11 @@ async def run_avito_detail_backfill(
     counters = AvitoDetailBackfillResult()
     current_counters: dict[str, int] = counters.to_dict()
 
-    browser_mode = settings.scraper_fetch_mode == "browser"
+    # Режим фетча: curl_cffi+backconnect (mproxy) когда use_curl=True; иначе браузер/auv.
+    # avito_detail_backfill_use_curl=True переопределяет scraper_fetch_mode — detail_backfill
+    # всегда идёт через backconnect чтобы не конкурировать за auv с SERP (city_sweep/full_load).
+    use_curl = settings.avito_detail_backfill_use_curl
+    browser_mode = settings.scraper_fetch_mode == "browser" and not use_curl
     session: AsyncSession | None = None
     browser_fetcher: BrowserFetcher | None = None
     own_session = False
@@ -98,14 +102,25 @@ async def run_avito_detail_backfill(
     scraper = AvitoScraper()
     start = time.monotonic()
 
+    logger.info(
+        "avito_detail_backfill: run_id=%d mode=%s",
+        run_id,
+        "curl/backconnect" if use_curl else "browser",
+    )
+
     try:
-        # Setup session (mirrors run_avito_pipeline lines 161-183)
+        # Setup session (mirrors run_avito_pipeline lines 161-183).
+        # use_curl=True: session=None + browser_fetcher=None → fetch_detail строит
+        # эфемерную _build_detail_session() через settings.scraper_proxy_url (backconnect).
+        # use_curl=False (legacy): browser_mode=True → BrowserFetcher/auv, как раньше.
         if browser_mode:
             browser_fetcher = BrowserFetcher(source="avito")
             await browser_fetcher.__aenter__()
             own_browser = True
             scraper._browser = browser_fetcher
-        else:
+        elif not use_curl:
+            # curl_cffi legacy path (scraper_fetch_mode="curl_cffi", use_curl=False):
+            # строим shared сессию через auv, как делал scrape_pipeline.
             own_session = True
             session = AsyncSession(
                 impersonate="chrome120",
@@ -114,6 +129,8 @@ async def run_avito_detail_backfill(
                 proxies=_avito_proxies(),
             )
             scraper._cffi = session
+        # use_curl=True: ничего не строим — fetch_detail вызывает _build_detail_session()
+        # (scraper_proxy_url = backconnect mproxy) на каждый запрос.
 
         runs_mod.update_heartbeat(db, run_id, current_counters)
 
@@ -156,13 +173,13 @@ async def run_avito_detail_backfill(
 
         logger.info(
             "avito_detail_backfill: run_id=%d snapshot=%d (budget=%.0fs "
-            "delay=%.1fs max_blocks=%d browser=%s)",
+            "delay=%.1fs max_blocks=%d mode=%s)",
             run_id,
             len(snapshot),
             budget_sec,
             request_delay_sec,
             max_consecutive_blocks,
-            browser_mode,
+            "curl/backconnect" if use_curl else "browser",
         )
 
         consecutive_blocks = 0

tradein-mvp/backend/tests/tasks/test_avito_detail_backfill.py

@@ -270,3 +270,88 @@ async def test_backfill_fetch_exception_continues() -> None:
     assert result.attempted == 2
     db.rollback.assert_called()
     runs.mark_done.assert_called_once()
+
+
+_BROWSER_FETCHER = "app.tasks.avito_detail_backfill.BrowserFetcher"
+
+
+@pytest.mark.asyncio
+async def test_backfill_use_curl_flag_skips_browser_fetcher() -> None:
+    """avito_detail_backfill_use_curl=True: BrowserFetcher не создаётся,
+    fetch_detail вызывается с browser_fetcher=None (curl/backconnect путь).
+    """
+    snapshot = _make_snapshot(1)
+    db = _mock_db(snapshot)
+    runs = MagicMock()
+    mock_enrichment = MagicMock()
+    mock_fetch = AsyncMock(return_value=mock_enrichment)
+    mock_save = MagicMock(return_value=True)
+    # Флаг use_curl=True, fetch_mode=browser (но флаг перекрывает)
+    fake_settings = MagicMock(
+        scraper_fetch_mode="browser",
+        avito_detail_backfill_use_curl=True,
+    )
+    with (
+        patch(_SETTINGS, fake_settings),
+        patch(_SESSION),
+        patch(_SCRAPER),
+        patch(_BROWSER_FETCHER) as mock_bf_cls,
+        patch(_RUNS, runs),
+        patch(_FETCH, mock_fetch),
+        patch(_SAVE, mock_save),
+        patch(_SLEEP, new_callable=AsyncMock),
+    ):
+        result = await run_avito_detail_backfill(
+            db, run_id=9, params={"batch_size": 10, "budget_sec": 3600}
+        )
+
+    # BrowserFetcher не должен быть создан
+    mock_bf_cls.assert_not_called()
+    # fetch_detail вызван с browser_fetcher=None (curl путь)
+    assert mock_fetch.call_count == 1
+    _, kwargs = mock_fetch.call_args
+    assert kwargs.get("browser_fetcher") is None
+    assert result.enriched == 1
+    runs.mark_done.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_backfill_use_curl_false_creates_browser_fetcher() -> None:
+    """avito_detail_backfill_use_curl=False + scraper_fetch_mode='browser':
+    BrowserFetcher создаётся (legacy browser/auv путь).
+    """
+    snapshot = _make_snapshot(1)
+    db = _mock_db(snapshot)
+    runs = MagicMock()
+    mock_enrichment = MagicMock()
+    mock_fetch = AsyncMock(return_value=mock_enrichment)
+    mock_save = MagicMock(return_value=True)
+    fake_settings = MagicMock(
+        scraper_fetch_mode="browser",
+        avito_detail_backfill_use_curl=False,
+    )
+    mock_bf_instance = AsyncMock()
+    mock_bf_instance.__aenter__ = AsyncMock(return_value=mock_bf_instance)
+    mock_bf_instance.__aexit__ = AsyncMock(return_value=False)
+    with (
+        patch(_SETTINGS, fake_settings),
+        patch(_SESSION),
+        patch(_SCRAPER),
+        patch(_BROWSER_FETCHER, return_value=mock_bf_instance) as mock_bf_cls,
+        patch(_RUNS, runs),
+        patch(_FETCH, mock_fetch),
+        patch(_SAVE, mock_save),
+        patch(_SLEEP, new_callable=AsyncMock),
+    ):
+        result = await run_avito_detail_backfill(
+            db, run_id=10, params={"batch_size": 10, "budget_sec": 3600}
+        )
+
+    # BrowserFetcher должен быть создан (source="avito")
+    mock_bf_cls.assert_called_once_with(source="avito")
+    # fetch_detail вызван с browser_fetcher установленным (не None)
+    assert mock_fetch.call_count == 1
+    _, kwargs = mock_fetch.call_args
+    assert kwargs.get("browser_fetcher") is not None
+    assert result.enriched == 1
+    runs.mark_done.assert_called_once()