From 2c6d0e4c71dd8e900fabdf2f3f34b422cefd6f4a Mon Sep 17 00:00:00 2001 From: bot-backend Date: Thu, 18 Jun 2026 19:24:05 +0300 Subject: [PATCH] feat(tradein): avito curl_cffi 429 short-retry (backconnect conn-limit transient, no rotate/ban) --- .../backend/app/services/scrapers/avito.py | 22 +++ .../tests/scrapers/test_avito_anti_bot.py | 136 +++++++++++++++++- 2 files changed, 155 insertions(+), 3 deletions(-) diff --git a/tradein-mvp/backend/app/services/scrapers/avito.py b/tradein-mvp/backend/app/services/scrapers/avito.py index f7f44d1a..c7a9bf4e 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito.py +++ b/tradein-mvp/backend/app/services/scrapers/avito.py @@ -248,6 +248,13 @@ _AVITO_PRICE_SEED_BRACKETS: list[tuple[int, int | None]] = get_price_seed_bracke # Avito SERP показывает ~50 карточек на страницу. _AVITO_OFFERS_PER_PAGE = 50 +# HTTP 429 в curl_cffi-режиме через backconnect-прокси (mproxy.site) — НЕ IP-ban, а +# transient «слишком много одновременных соединений» (лимит 5). Проходит на коротком +# retry без ротации IP. Делаем до _AVITO_429_MAX_RETRIES коротких пауз перед тем как +# свалиться в старое поведение (rotation / AvitoRateLimitedError). +_AVITO_429_MAX_RETRIES = 4 +_AVITO_429_BACKOFF_SEC = 1.5 + # Счётчик результатов в шапке SERP: 1 178. # Текст может содержать NBSP (\xa0),  -entity или обычные пробелы как разделители # разрядов — чистятся в _extract_total_count. [^<]* нежадно ловит весь текст до . @@ -394,6 +401,7 @@ class AvitoScraper(BaseScraper): max_rot = settings.avito_proxy_max_rotations if settings.avito_proxy_rotate_url else 0 rot_done = 0 transient_left = 2 + r429 = 0 while True: try: response = await self._cffi.get(url) @@ -412,6 +420,20 @@ class AvitoScraper(BaseScraper): sc = response.status_code is_firewall = sc == 200 and _is_firewall_page(response.text) + # 429 в curl_cffi через backconnect-прокси — transient conn-limit, НЕ + # IP-ban: короткий retry того же url без ротации. Только если короткие + # retry исчерпаны — проваливаемся в общую блок-логику ниже (rotation/raise). + if sc == 429 and r429 < _AVITO_429_MAX_RETRIES: + r429 += 1 + logger.info( + "avito page=%d HTTP 429 (backconnect conn-limit) — short retry %d/%d", + page, + r429, + _AVITO_429_MAX_RETRIES, + ) + jitter = (r429 % 3) * 0.1 + await asyncio.sleep(_AVITO_429_BACKOFF_SEC + jitter) + continue if sc in (403, 429) or is_firewall: if rot_done < max_rot and await self._rotate_ip(): rot_done += 1 diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py b/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py index c94e7e03..f6ebb3d8 100644 --- a/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py +++ b/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py @@ -151,7 +151,11 @@ async def test_fetch_around_raises_blocked_on_403() -> None: @pytest.mark.asyncio async def test_fetch_around_raises_rate_limited_on_429() -> None: - """AvitoScraper.fetch_around должен raise AvitoRateLimitedError на HTTP 429.""" + """AvitoScraper.fetch_around должен raise AvitoRateLimitedError на persistent HTTP 429. + + 429 (backconnect conn-limit) сначала проходит короткие retry; при отсутствии + rotate-url после исчерпания retry сохраняется старый fallback — AvitoRateLimitedError. + """ from app.services.scrapers.avito import AvitoScraper scraper = AvitoScraper() @@ -161,8 +165,9 @@ async def test_fetch_around_raises_rate_limited_on_429() -> None: mock_session.get = AsyncMock(return_value=mock_response) scraper._cffi = mock_session - with pytest.raises(AvitoRateLimitedError): - await scraper.fetch_around(56.84, 60.60, 1000, pages=1) + with patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()): + with pytest.raises(AvitoRateLimitedError): + await scraper.fetch_around(56.84, 60.60, 1000, pages=1) # ── Pipeline abort tests ──────────────────────────────────────── @@ -431,3 +436,128 @@ def test_avito_proxies_helper(monkeypatch: pytest.MonkeyPatch) -> None: url = "http://u:p@mproxy.site:16928" monkeypatch.setattr(sp.settings, "avito_proxy_url", url) assert sp._avito_proxies() == {"http": url, "https": url} + + +# ── curl_cffi 429 short-retry (backconnect conn-limit transient) ────────────── + + +@pytest.mark.asyncio +async def test_fetch_around_429_short_retry_then_succeeds() -> None: + """429 дважды → 200: короткий retry снимает transient conn-limit БЕЗ ротации IP. + + backconnect-прокси (mproxy.site, лимит 5 соединений) отдаёт 429 при параллельной + пагинации — это transient, проходит на коротком retry. _rotate_ip НЕ должен дёргаться. + """ + from app.services.scrapers.avito import AvitoScraper + + scraper = AvitoScraper() + r429 = MagicMock() + r429.status_code = 429 + ok = MagicMock() + ok.status_code = 200 + ok.text = _CARD_HTML + mock_session = AsyncMock() + mock_session.get = AsyncMock(side_effect=[r429, r429, ok]) + scraper._cffi = mock_session + + with ( + patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()) as slp, + patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot, + patch.object(AvitoScraper, "sleep_between_requests", AsyncMock()), + ): + result = await scraper.fetch_around(56.84, 60.60, 1000, pages=1) + + assert len(result) == 1 + assert mock_session.get.await_count == 3 # 429, 429, 200 + rot.assert_not_awaited() # короткий retry, не ротация + # короткий backoff (_AVITO_429_BACKOFF_SEC ~1.5s), не 9с-settle ротации + assert slp.await_count >= 2 + for c in slp.await_args_list: + assert c.args[0] < 5.0 + + +@pytest.mark.asyncio +async def test_fetch_around_429_exhausted_no_rotate_url_raises() -> None: + """429 постоянно + НЕТ rotate-url → после исчерпания коротких retry: старый fallback.""" + from app.services.scrapers import avito as avito_mod + from app.services.scrapers.avito import AvitoScraper + + scraper = AvitoScraper() + r429 = MagicMock() + r429.status_code = 429 + mock_session = AsyncMock() + mock_session.get = AsyncMock(return_value=r429) + scraper._cffi = mock_session + + with ( + patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()) as slp, + patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot, + ): + with pytest.raises(AvitoRateLimitedError): + await scraper.fetch_around(56.84, 60.60, 1000, pages=1) + + rot.assert_not_awaited() # нет rotate-url → ротация не дёргается + # ровно _AVITO_429_MAX_RETRIES коротких retry, потом fallback + assert slp.await_count == avito_mod._AVITO_429_MAX_RETRIES + + +@pytest.mark.asyncio +async def test_fetch_around_429_exhausted_with_rotate_url_rotates( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """429 постоянно + ЕСТЬ rotate-url → короткие retry, затем ротация (старый путь).""" + from app.services.scrapers import avito as avito_mod + from app.services.scrapers.avito import AvitoScraper + + monkeypatch.setattr( + avito_mod.settings, "avito_proxy_rotate_url", "https://changeip.example/?proxy_key=x" + ) + monkeypatch.setattr(avito_mod.settings, "avito_proxy_max_rotations", 1) + + scraper = AvitoScraper() + r429 = MagicMock() + r429.status_code = 429 + ok = MagicMock() + ok.status_code = 200 + ok.text = _CARD_HTML + mock_session = AsyncMock() + # 429 на коротких retry, после ротации — 200 + mock_session.get = AsyncMock(side_effect=[r429, r429, r429, r429, r429, ok]) + scraper._cffi = mock_session + + with ( + patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()), + patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot, + patch.object(AvitoScraper, "sleep_between_requests", AsyncMock()), + ): + result = await scraper.fetch_around(56.84, 60.60, 1000, pages=1) + + rot.assert_awaited_once() # после исчерпания коротких retry — ротация + assert len(result) == 1 + + +@pytest.mark.asyncio +async def test_fetch_around_403_no_429_retry_path(monkeypatch: pytest.MonkeyPatch) -> None: + """403 (реальный IP-ban) НЕ проходит через 429-retry — поведение без изменений.""" + from app.services.scrapers import avito as avito_mod + from app.services.scrapers.avito import AvitoScraper + + monkeypatch.setattr(avito_mod.settings, "avito_proxy_rotate_url", None) + + scraper = AvitoScraper() + r403 = MagicMock() + r403.status_code = 403 + mock_session = AsyncMock() + mock_session.get = AsyncMock(return_value=r403) + scraper._cffi = mock_session + + with ( + patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()) as slp, + patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)), + ): + with pytest.raises(AvitoBlockedError): + await scraper.fetch_around(56.84, 60.60, 1000, pages=1) + + # 403 без rotate-url → немедленный raise, ни одного 429-backoff sleep + slp.assert_not_awaited() + assert mock_session.get.await_count == 1