feat(tradein): avito curl_cffi 429 short-retry (backconnect conn-limit transient, no rotate/ban)

tradein-mvp/backend/app/services/scrapers/avito.py

@@ -248,6 +248,13 @@ _AVITO_PRICE_SEED_BRACKETS: list[tuple[int, int | None]] = get_price_seed_bracke
 # Avito SERP показывает ~50 карточек на страницу.
 _AVITO_OFFERS_PER_PAGE = 50
 
+# HTTP 429 в curl_cffi-режиме через backconnect-прокси (mproxy.site) — НЕ IP-ban, а
+# transient «слишком много одновременных соединений» (лимит 5). Проходит на коротком
+# retry без ротации IP. Делаем до _AVITO_429_MAX_RETRIES коротких пауз перед тем как
+# свалиться в старое поведение (rotation / AvitoRateLimitedError).
+_AVITO_429_MAX_RETRIES = 4
+_AVITO_429_BACKOFF_SEC = 1.5
+
 # Счётчик результатов в шапке SERP: <span data-marker="page-title/count">1&nbsp;178</span>.
 # Текст может содержать NBSP (\xa0), &nbsp;-entity или обычные пробелы как разделители
 # разрядов — чистятся в _extract_total_count. [^<]* нежадно ловит весь текст до </span>.
@@ -394,6 +401,7 @@ class AvitoScraper(BaseScraper):
         max_rot = settings.avito_proxy_max_rotations if settings.avito_proxy_rotate_url else 0
         rot_done = 0
         transient_left = 2
+        r429 = 0
         while True:
             try:
                 response = await self._cffi.get(url)
@@ -412,6 +420,20 @@ class AvitoScraper(BaseScraper):
 
             sc = response.status_code
             is_firewall = sc == 200 and _is_firewall_page(response.text)
+            # 429 в curl_cffi через backconnect-прокси — transient conn-limit, НЕ
+            # IP-ban: короткий retry того же url без ротации. Только если короткие
+            # retry исчерпаны — проваливаемся в общую блок-логику ниже (rotation/raise).
+            if sc == 429 and r429 < _AVITO_429_MAX_RETRIES:
+                r429 += 1
+                logger.info(
+                    "avito page=%d HTTP 429 (backconnect conn-limit) — short retry %d/%d",
+                    page,
+                    r429,
+                    _AVITO_429_MAX_RETRIES,
+                )
+                jitter = (r429 % 3) * 0.1
+                await asyncio.sleep(_AVITO_429_BACKOFF_SEC + jitter)
+                continue
             if sc in (403, 429) or is_firewall:
                 if rot_done < max_rot and await self._rotate_ip():
                     rot_done += 1

tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py

@@ -151,7 +151,11 @@ async def test_fetch_around_raises_blocked_on_403() -> None:
 
 @pytest.mark.asyncio
 async def test_fetch_around_raises_rate_limited_on_429() -> None:
-    """AvitoScraper.fetch_around должен raise AvitoRateLimitedError на HTTP 429."""
+    """AvitoScraper.fetch_around должен raise AvitoRateLimitedError на persistent HTTP 429.
+
+    429 (backconnect conn-limit) сначала проходит короткие retry; при отсутствии
+    rotate-url после исчерпания retry сохраняется старый fallback — AvitoRateLimitedError.
+    """
     from app.services.scrapers.avito import AvitoScraper
 
     scraper = AvitoScraper()
@@ -161,8 +165,9 @@ async def test_fetch_around_raises_rate_limited_on_429() -> None:
     mock_session.get = AsyncMock(return_value=mock_response)
     scraper._cffi = mock_session
 
-    with pytest.raises(AvitoRateLimitedError):
+    with patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()):
-        await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
+        with pytest.raises(AvitoRateLimitedError):
+            await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
 
 
 # ── Pipeline abort tests ────────────────────────────────────────
@@ -431,3 +436,128 @@ def test_avito_proxies_helper(monkeypatch: pytest.MonkeyPatch) -> None:
     url = "http://u:p@mproxy.site:16928"
     monkeypatch.setattr(sp.settings, "avito_proxy_url", url)
     assert sp._avito_proxies() == {"http": url, "https": url}
+
+
+# ── curl_cffi 429 short-retry (backconnect conn-limit transient) ──────────────
+
+
+@pytest.mark.asyncio
+async def test_fetch_around_429_short_retry_then_succeeds() -> None:
+    """429 дважды → 200: короткий retry снимает transient conn-limit БЕЗ ротации IP.
+
+    backconnect-прокси (mproxy.site, лимит 5 соединений) отдаёт 429 при параллельной
+    пагинации — это transient, проходит на коротком retry. _rotate_ip НЕ должен дёргаться.
+    """
+    from app.services.scrapers.avito import AvitoScraper
+
+    scraper = AvitoScraper()
+    r429 = MagicMock()
+    r429.status_code = 429
+    ok = MagicMock()
+    ok.status_code = 200
+    ok.text = _CARD_HTML
+    mock_session = AsyncMock()
+    mock_session.get = AsyncMock(side_effect=[r429, r429, ok])
+    scraper._cffi = mock_session
+
+    with (
+        patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()) as slp,
+        patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot,
+        patch.object(AvitoScraper, "sleep_between_requests", AsyncMock()),
+    ):
+        result = await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
+
+    assert len(result) == 1
+    assert mock_session.get.await_count == 3  # 429, 429, 200
+    rot.assert_not_awaited()  # короткий retry, не ротация
+    # короткий backoff (_AVITO_429_BACKOFF_SEC ~1.5s), не 9с-settle ротации
+    assert slp.await_count >= 2
+    for c in slp.await_args_list:
+        assert c.args[0] < 5.0
+
+
+@pytest.mark.asyncio
+async def test_fetch_around_429_exhausted_no_rotate_url_raises() -> None:
+    """429 постоянно + НЕТ rotate-url → после исчерпания коротких retry: старый fallback."""
+    from app.services.scrapers import avito as avito_mod
+    from app.services.scrapers.avito import AvitoScraper
+
+    scraper = AvitoScraper()
+    r429 = MagicMock()
+    r429.status_code = 429
+    mock_session = AsyncMock()
+    mock_session.get = AsyncMock(return_value=r429)
+    scraper._cffi = mock_session
+
+    with (
+        patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()) as slp,
+        patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot,
+    ):
+        with pytest.raises(AvitoRateLimitedError):
+            await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
+
+    rot.assert_not_awaited()  # нет rotate-url → ротация не дёргается
+    # ровно _AVITO_429_MAX_RETRIES коротких retry, потом fallback
+    assert slp.await_count == avito_mod._AVITO_429_MAX_RETRIES
+
+
+@pytest.mark.asyncio
+async def test_fetch_around_429_exhausted_with_rotate_url_rotates(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """429 постоянно + ЕСТЬ rotate-url → короткие retry, затем ротация (старый путь)."""
+    from app.services.scrapers import avito as avito_mod
+    from app.services.scrapers.avito import AvitoScraper
+
+    monkeypatch.setattr(
+        avito_mod.settings, "avito_proxy_rotate_url", "https://changeip.example/?proxy_key=x"
+    )
+    monkeypatch.setattr(avito_mod.settings, "avito_proxy_max_rotations", 1)
+
+    scraper = AvitoScraper()
+    r429 = MagicMock()
+    r429.status_code = 429
+    ok = MagicMock()
+    ok.status_code = 200
+    ok.text = _CARD_HTML
+    mock_session = AsyncMock()
+    # 429 на коротких retry, после ротации — 200
+    mock_session.get = AsyncMock(side_effect=[r429, r429, r429, r429, r429, ok])
+    scraper._cffi = mock_session
+
+    with (
+        patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()),
+        patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot,
+        patch.object(AvitoScraper, "sleep_between_requests", AsyncMock()),
+    ):
+        result = await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
+
+    rot.assert_awaited_once()  # после исчерпания коротких retry — ротация
+    assert len(result) == 1
+
+
+@pytest.mark.asyncio
+async def test_fetch_around_403_no_429_retry_path(monkeypatch: pytest.MonkeyPatch) -> None:
+    """403 (реальный IP-ban) НЕ проходит через 429-retry — поведение без изменений."""
+    from app.services.scrapers import avito as avito_mod
+    from app.services.scrapers.avito import AvitoScraper
+
+    monkeypatch.setattr(avito_mod.settings, "avito_proxy_rotate_url", None)
+
+    scraper = AvitoScraper()
+    r403 = MagicMock()
+    r403.status_code = 403
+    mock_session = AsyncMock()
+    mock_session.get = AsyncMock(return_value=r403)
+    scraper._cffi = mock_session
+
+    with (
+        patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()) as slp,
+        patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)),
+    ):
+        with pytest.raises(AvitoBlockedError):
+            await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
+
+    # 403 без rotate-url → немедленный raise, ни одного 429-backoff sleep
+    slp.assert_not_awaited()
+    assert mock_session.get.await_count == 1