fix(avito): proactive IP rotate at sweep start + raise max_rotations 2->4 (Datadome ban recovery)

tradein-mvp/backend/app/core/config.py

@@ -208,7 +208,9 @@ class Settings(BaseSettings):
     # ENV: AVITO_PROXY_ROTATE_URL.
     avito_proxy_rotate_url: str | None = None
     # Сколько раз сменить IP при блоке прежде чем сдаться (на одну страницу).
-    avito_proxy_max_rotations: int = 2
+    # #1731: 2→4 — больше шансов восстановиться mid-sweep после проактивной
+    # ротации на старте (Datadome ban recovery).
+    avito_proxy_max_rotations: int = 4
 
     # ── Cian dedicated mobile proxy (separate egress from Avito) ──────────────
     # Cian и Avito делят один мобильный IP при общем scraper_proxy_url → конкуренция

tradein-mvp/backend/app/services/scrapers/avito.py

@@ -244,6 +244,22 @@ class AvitoScraper(BaseScraper):
         self.parse_failures: int = 0
 
     async def __aenter__(self) -> AvitoScraper:
+        # Проактивная ротация IP в начале sweep (#1731): apw-IP «протухает» по
+        # Datadome между суточными прогонами, поэтому первые якоря блокируются и
+        # реактивного бюджета max_rotations не хватает (3 подряд блока →
+        # mark_banned). Один changeip-вызов до первого SERP-fetch даёт свежий IP.
+        # _rotate_ip() сам ждёт ~9с (settle) и логирует/глотает ошибку, поэтому
+        # тут только дополнительный guard: сбой ротации НЕ должен ронять sweep.
+        if settings.avito_proxy_rotate_url:
+            try:
+                if await self._rotate_ip():
+                    logger.info("avito: proactive IP rotate at sweep start (#1731)")
+                else:
+                    logger.warning("avito: proactive IP rotate at sweep start failed — proceeding")
+            except Exception:
+                logger.warning(
+                    "avito: proactive IP rotate at sweep start raised — proceeding", exc_info=True
+                )
         if settings.scraper_fetch_mode == "browser":
             self._browser = BrowserFetcher()
             await self._browser.__aenter__()

tradein-mvp/backend/tests/test_avito_browser_mode.py

@@ -72,6 +72,83 @@ async def test_aenter_browser_mode_sets_browser_attribute() -> None:
     assert scraper._cffi is None
 
 
+@pytest.mark.asyncio
+async def test_aenter_proactive_rotate_when_rotate_url_set() -> None:
+    """#1731: rotate_url configured → __aenter__ calls _rotate_ip exactly once at start.
+
+    _rotate_ip is mocked (no real changeip HTTP / no 9s sleep). Browser mode used
+    to keep BaseScraper setup out of the picture.
+    """
+    mock_browser = _make_mock_browser()
+    scraper = AvitoScraper()
+    mock_rotate = AsyncMock(return_value=True)
+
+    with patch("app.services.scrapers.avito.settings") as mock_settings:
+        mock_settings.scraper_fetch_mode = "browser"
+        mock_settings.scraper_proxy_url = None
+        mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/changeip"
+        mock_settings.avito_proxy_max_rotations = 4
+
+        with patch(
+            "app.services.scrapers.avito.BrowserFetcher",
+            return_value=mock_browser,
+        ):
+            with patch.object(scraper, "_rotate_ip", new=mock_rotate):
+                result = await scraper.__aenter__()
+
+    assert result is scraper
+    mock_rotate.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_aenter_no_proactive_rotate_when_rotate_url_none() -> None:
+    """#1731: rotate_url is None → __aenter__ must NOT call _rotate_ip."""
+    mock_browser = _make_mock_browser()
+    scraper = AvitoScraper()
+    mock_rotate = AsyncMock(return_value=True)
+
+    with patch("app.services.scrapers.avito.settings") as mock_settings:
+        mock_settings.scraper_fetch_mode = "browser"
+        mock_settings.scraper_proxy_url = None
+        mock_settings.avito_proxy_rotate_url = None
+        mock_settings.avito_proxy_max_rotations = 4
+
+        with patch(
+            "app.services.scrapers.avito.BrowserFetcher",
+            return_value=mock_browser,
+        ):
+            with patch.object(scraper, "_rotate_ip", new=mock_rotate):
+                await scraper.__aenter__()
+
+    mock_rotate.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_aenter_proactive_rotate_failure_does_not_abort() -> None:
+    """#1731: a failed proactive rotate (returns False / raises) must NOT abort __aenter__."""
+    mock_browser = _make_mock_browser()
+    scraper = AvitoScraper()
+
+    with patch("app.services.scrapers.avito.settings") as mock_settings:
+        mock_settings.scraper_fetch_mode = "browser"
+        mock_settings.scraper_proxy_url = None
+        mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/changeip"
+        mock_settings.avito_proxy_max_rotations = 4
+
+        with patch(
+            "app.services.scrapers.avito.BrowserFetcher",
+            return_value=mock_browser,
+        ):
+            # _rotate_ip raises — sweep must proceed (browser still attached)
+            with patch.object(
+                scraper, "_rotate_ip", new=AsyncMock(side_effect=RuntimeError("changeip down"))
+            ):
+                result = await scraper.__aenter__()
+
+    assert result is scraper
+    assert scraper._browser is mock_browser
+
+
 @pytest.mark.asyncio
 async def test_fetch_serp_html_browser_mode_returns_html() -> None:
     """_fetch_serp_html with _browser set returns HTML string, never touches _cffi."""

tradein-mvp/backend/tests/test_config_avito_rotations.py

@@ -0,0 +1,17 @@
+"""#1731 — default avito_proxy_max_rotations bumped 2→4 (Datadome ban recovery).
+
+Reactive rotation budget per page is raised so a sweep has more chances to recover
+mid-run after the proactive rotate-at-start. No network, no DB.
+"""
+
+import os
+
+os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
+
+from app.core.config import Settings
+
+
+def test_avito_proxy_max_rotations_default_is_4() -> None:
+    """Field default must be 4 (was 2 before #1731)."""
+    s = Settings()
+    assert s.avito_proxy_max_rotations == 4