diff --git a/tradein-mvp/backend/app/core/config.py b/tradein-mvp/backend/app/core/config.py index e662cbca..8c0826e4 100644 --- a/tradein-mvp/backend/app/core/config.py +++ b/tradein-mvp/backend/app/core/config.py @@ -208,7 +208,9 @@ class Settings(BaseSettings): # ENV: AVITO_PROXY_ROTATE_URL. avito_proxy_rotate_url: str | None = None # Сколько раз сменить IP при блоке прежде чем сдаться (на одну страницу). - avito_proxy_max_rotations: int = 2 + # #1731: 2→4 — больше шансов восстановиться mid-sweep после проактивной + # ротации на старте (Datadome ban recovery). + avito_proxy_max_rotations: int = 4 # ── Cian dedicated mobile proxy (separate egress from Avito) ────────────── # Cian и Avito делят один мобильный IP при общем scraper_proxy_url → конкуренция diff --git a/tradein-mvp/backend/app/services/scrapers/avito.py b/tradein-mvp/backend/app/services/scrapers/avito.py index 4edb6281..2a1ed86b 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito.py +++ b/tradein-mvp/backend/app/services/scrapers/avito.py @@ -244,6 +244,22 @@ class AvitoScraper(BaseScraper): self.parse_failures: int = 0 async def __aenter__(self) -> AvitoScraper: + # Проактивная ротация IP в начале sweep (#1731): apw-IP «протухает» по + # Datadome между суточными прогонами, поэтому первые якоря блокируются и + # реактивного бюджета max_rotations не хватает (3 подряд блока → + # mark_banned). Один changeip-вызов до первого SERP-fetch даёт свежий IP. + # _rotate_ip() сам ждёт ~9с (settle) и логирует/глотает ошибку, поэтому + # тут только дополнительный guard: сбой ротации НЕ должен ронять sweep. + if settings.avito_proxy_rotate_url: + try: + if await self._rotate_ip(): + logger.info("avito: proactive IP rotate at sweep start (#1731)") + else: + logger.warning("avito: proactive IP rotate at sweep start failed — proceeding") + except Exception: + logger.warning( + "avito: proactive IP rotate at sweep start raised — proceeding", exc_info=True + ) if settings.scraper_fetch_mode == "browser": self._browser = BrowserFetcher() await self._browser.__aenter__() diff --git a/tradein-mvp/backend/tests/test_avito_browser_mode.py b/tradein-mvp/backend/tests/test_avito_browser_mode.py index f6be8369..75e453be 100644 --- a/tradein-mvp/backend/tests/test_avito_browser_mode.py +++ b/tradein-mvp/backend/tests/test_avito_browser_mode.py @@ -72,6 +72,83 @@ async def test_aenter_browser_mode_sets_browser_attribute() -> None: assert scraper._cffi is None +@pytest.mark.asyncio +async def test_aenter_proactive_rotate_when_rotate_url_set() -> None: + """#1731: rotate_url configured → __aenter__ calls _rotate_ip exactly once at start. + + _rotate_ip is mocked (no real changeip HTTP / no 9s sleep). Browser mode used + to keep BaseScraper setup out of the picture. + """ + mock_browser = _make_mock_browser() + scraper = AvitoScraper() + mock_rotate = AsyncMock(return_value=True) + + with patch("app.services.scrapers.avito.settings") as mock_settings: + mock_settings.scraper_fetch_mode = "browser" + mock_settings.scraper_proxy_url = None + mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/changeip" + mock_settings.avito_proxy_max_rotations = 4 + + with patch( + "app.services.scrapers.avito.BrowserFetcher", + return_value=mock_browser, + ): + with patch.object(scraper, "_rotate_ip", new=mock_rotate): + result = await scraper.__aenter__() + + assert result is scraper + mock_rotate.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_aenter_no_proactive_rotate_when_rotate_url_none() -> None: + """#1731: rotate_url is None → __aenter__ must NOT call _rotate_ip.""" + mock_browser = _make_mock_browser() + scraper = AvitoScraper() + mock_rotate = AsyncMock(return_value=True) + + with patch("app.services.scrapers.avito.settings") as mock_settings: + mock_settings.scraper_fetch_mode = "browser" + mock_settings.scraper_proxy_url = None + mock_settings.avito_proxy_rotate_url = None + mock_settings.avito_proxy_max_rotations = 4 + + with patch( + "app.services.scrapers.avito.BrowserFetcher", + return_value=mock_browser, + ): + with patch.object(scraper, "_rotate_ip", new=mock_rotate): + await scraper.__aenter__() + + mock_rotate.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_aenter_proactive_rotate_failure_does_not_abort() -> None: + """#1731: a failed proactive rotate (returns False / raises) must NOT abort __aenter__.""" + mock_browser = _make_mock_browser() + scraper = AvitoScraper() + + with patch("app.services.scrapers.avito.settings") as mock_settings: + mock_settings.scraper_fetch_mode = "browser" + mock_settings.scraper_proxy_url = None + mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/changeip" + mock_settings.avito_proxy_max_rotations = 4 + + with patch( + "app.services.scrapers.avito.BrowserFetcher", + return_value=mock_browser, + ): + # _rotate_ip raises — sweep must proceed (browser still attached) + with patch.object( + scraper, "_rotate_ip", new=AsyncMock(side_effect=RuntimeError("changeip down")) + ): + result = await scraper.__aenter__() + + assert result is scraper + assert scraper._browser is mock_browser + + @pytest.mark.asyncio async def test_fetch_serp_html_browser_mode_returns_html() -> None: """_fetch_serp_html with _browser set returns HTML string, never touches _cffi.""" diff --git a/tradein-mvp/backend/tests/test_config_avito_rotations.py b/tradein-mvp/backend/tests/test_config_avito_rotations.py new file mode 100644 index 00000000..219d6598 --- /dev/null +++ b/tradein-mvp/backend/tests/test_config_avito_rotations.py @@ -0,0 +1,17 @@ +"""#1731 — default avito_proxy_max_rotations bumped 2→4 (Datadome ban recovery). + +Reactive rotation budget per page is raised so a sweep has more chances to recover +mid-run after the proactive rotate-at-start. No network, no DB. +""" + +import os + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") + +from app.core.config import Settings + + +def test_avito_proxy_max_rotations_default_is_4() -> None: + """Field default must be 4 (was 2 before #1731).""" + s = Settings() + assert s.avito_proxy_max_rotations == 4