tradein-mvp/backend/app/services/scrapers/avito.py

@@ -41,6 +41,7 @@ from app.services.scrapers.avito_exceptions import (
     AvitoRateLimitedError,
 )
 from app.services.scrapers.base import BaseScraper, ScrapedLot
+from app.services.scrapers.browser_fetcher import BrowserFetcher
 
 logger = logging.getLogger(__name__)
 
@@ -215,10 +216,16 @@ class AvitoScraper(BaseScraper):
         super().__init__()
         self.request_delay_sec = get_scraper_delay(self.name)
         self._cffi: AsyncSession | None = None
+        self._browser: BrowserFetcher | None = None
         # #823: счётчик карточек, которые не удалось распарсить из-за неожиданной структуры DOM.
         self.parse_failures: int = 0
 
     async def __aenter__(self) -> AvitoScraper:
+        if settings.scraper_fetch_mode == "browser":
+            self._browser = BrowserFetcher()
+            await self._browser.__aenter__()
+            logger.info("avito: SERP fetch via BrowserFetcher (camoufox) — #901")
+            return self
         await super().__aenter__()
         proxies = None
         if settings.scraper_proxy_url:
@@ -242,6 +249,8 @@ class AvitoScraper(BaseScraper):
         return self
 
     async def __aexit__(self, *args: Any) -> None:
+        if self._browser is not None:
+            await self._browser.__aexit__(*args)
         if self._cffi is not None:
             await self._cffi.close()
         await super().__aexit__(*args)
@@ -277,9 +286,31 @@ class AvitoScraper(BaseScraper):
 
         Returns:
             HTML при HTTP 200, либо ``None`` при non-200 (конец пагинации).
+            В browser-mode HTTP-статус недоступен: метод возвращает HTML или
+            поднимает ``AvitoBlockedError`` (pagination ends via empty-parse in callers).
         Raises:
             AvitoBlockedError / AvitoRateLimitedError — если бан не снят ротацией.
         """
+        if self._browser is not None:
+            # Browser-mode (#901): no HTTP status code — soft firewall-detect + rotate.
+            max_rot = settings.avito_proxy_max_rotations if settings.avito_proxy_rotate_url else 0
+            rot_done = 0
+            while True:
+                html = await self._browser.fetch(url)
+                if _is_firewall_page(html):
+                    if rot_done < max_rot and await self._rotate_ip():
+                        rot_done += 1
+                        logger.info(
+                            "avito page=%d browser firewall — retry after rotation #%d",
+                            page,
+                            rot_done,
+                        )
+                        continue
+                    logger.error("avito SERP firewall in browser-mode page=%d url=%s", page, url)
+                    raise AvitoBlockedError(
+                        f"Avito SERP firewall (browser-mode) at page={page} — IP banned"
+                    )
+                return html
         assert self._cffi is not None
         max_rot = settings.avito_proxy_max_rotations if settings.avito_proxy_rotate_url else 0
         rot_done = 0

tradein-mvp/backend/tests/test_avito_browser_mode.py

@@ -0,0 +1,225 @@
+"""#901 — avito SERP fetch via BrowserFetcher behind scraper_fetch_mode flag.
+
+Tests:
+1. browser branch chosen: mode="browser" → _browser set, _cffi is None, HTML from mock browser.
+2. curl branch unchanged: default mode → _browser is None, _cffi session used.
+3. firewall-detect in browser-mode: firewall HTML + no rotate URL → AvitoBlockedError.
+4. pagination terminates via empty-parse: browser HTML with 0 lots → caller loop breaks.
+
+Без сети, без БД, без curl_cffi.
+"""
+
+from __future__ import annotations
+
+import os
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
+
+from app.services.scrapers.avito import _FIREWALL_MARKERS, AvitoScraper
+from app.services.scrapers.avito_exceptions import AvitoBlockedError
+from app.services.scrapers.base import ScrapedLot
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+_NORMAL_HTML = "<html><body>normal avito SERP</body></html>"
+_FIREWALL_HTML = f"<html><head><title>{_FIREWALL_MARKERS[0]}</title></head></html>"
+
+
+def _make_mock_browser(html: str = _NORMAL_HTML) -> MagicMock:
+    """Return a mock BrowserFetcher context manager that yields html from fetch()."""
+    mock = MagicMock()
+    mock.__aenter__ = AsyncMock(return_value=mock)
+    mock.__aexit__ = AsyncMock(return_value=None)
+    mock.fetch = AsyncMock(return_value=html)
+    return mock
+
+
+# ---------------------------------------------------------------------------
+# 1. browser branch chosen
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_aenter_browser_mode_sets_browser_attribute() -> None:
+    """mode=browser → __aenter__ instantiates BrowserFetcher, _cffi stays None."""
+    mock_browser = _make_mock_browser()
+    scraper = AvitoScraper()
+
+    with patch("app.services.scrapers.avito.settings") as mock_settings:
+        mock_settings.scraper_fetch_mode = "browser"
+        mock_settings.scraper_proxy_url = None
+        mock_settings.avito_proxy_rotate_url = None
+        mock_settings.avito_proxy_max_rotations = 2
+
+        with patch(
+            "app.services.scrapers.avito.BrowserFetcher",
+            return_value=mock_browser,
+        ):
+            # Patch BaseScraper.__aenter__ so we skip its real setup (DB/network)
+            with patch(
+                "app.services.scrapers.base.BaseScraper.__aenter__",
+                new=AsyncMock(return_value=scraper),
+            ):
+                result = await scraper.__aenter__()
+
+    assert result is scraper
+    assert scraper._browser is mock_browser
+    assert scraper._cffi is None
+
+
+@pytest.mark.asyncio
+async def test_fetch_serp_html_browser_mode_returns_html() -> None:
+    """_fetch_serp_html with _browser set returns HTML string, never touches _cffi."""
+    scraper = AvitoScraper()
+    mock_browser = _make_mock_browser(_NORMAL_HTML)
+    scraper._browser = mock_browser
+
+    result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
+
+    assert result == _NORMAL_HTML
+    mock_browser.fetch.assert_awaited_once_with("https://www.avito.ru/test")
+    # _cffi was never set — assert remains None
+    assert scraper._cffi is None
+
+
+# ---------------------------------------------------------------------------
+# 2. curl branch unchanged
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_fetch_serp_html_curl_mode_uses_cffi() -> None:
+    """Default mode (_browser is None) → _fetch_serp_html goes through curl path."""
+    scraper = AvitoScraper()
+
+    # Mock the curl session returning HTTP 200 with normal HTML
+    mock_response = MagicMock()
+    mock_response.status_code = 200
+    mock_response.text = _NORMAL_HTML
+
+    mock_cffi = MagicMock()
+    mock_cffi.get = AsyncMock(return_value=mock_response)
+    scraper._cffi = mock_cffi
+
+    # _browser must be None (default curl path)
+    assert scraper._browser is None
+
+    result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
+
+    assert result == _NORMAL_HTML
+    mock_cffi.get.assert_awaited_once_with("https://www.avito.ru/test")
+
+
+# ---------------------------------------------------------------------------
+# 3. firewall-detect in browser-mode → AvitoBlockedError (no rotate URL)
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_browser_mode_firewall_raises_blocked_error() -> None:
+    """Browser returns firewall page + no rotate URL configured → AvitoBlockedError."""
+    scraper = AvitoScraper()
+    mock_browser = _make_mock_browser(_FIREWALL_HTML)
+    scraper._browser = mock_browser
+
+    with patch("app.services.scrapers.avito.settings") as mock_settings:
+        mock_settings.avito_proxy_rotate_url = None  # no rotation configured
+        mock_settings.avito_proxy_max_rotations = 2
+
+        with pytest.raises(AvitoBlockedError, match="browser-mode"):
+            await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
+
+
+@pytest.mark.asyncio
+async def test_browser_mode_firewall_rotates_and_retries() -> None:
+    """Browser returns firewall on first call, normal HTML after rotation → returns HTML."""
+    scraper = AvitoScraper()
+
+    # First call → firewall; second call → normal HTML
+    mock_browser = MagicMock()
+    mock_browser.fetch = AsyncMock(side_effect=[_FIREWALL_HTML, _NORMAL_HTML])
+    scraper._browser = mock_browser
+
+    with patch("app.services.scrapers.avito.settings") as mock_settings:
+        mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/rotate"
+        mock_settings.avito_proxy_max_rotations = 1
+
+        with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=True)):
+            result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=2)
+
+    assert result == _NORMAL_HTML
+    assert mock_browser.fetch.await_count == 2
+
+
+# ---------------------------------------------------------------------------
+# 4. pagination terminates via empty-parse
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_fetch_city_wide_browser_mode_stops_on_empty_parse() -> None:
+    """fetch_city_wide with browser mode: page=1 returns lots, page=2 returns [] → stops.
+
+    Verifies the documented invariant: callers break on `not lots` regardless of
+    fetch mode. _fetch_serp_html is mocked at method level (browser vs curl is
+    already covered by unit tests above); here we confirm the caller loop behaviour.
+    """
+    scraper = AvitoScraper()
+
+    html_p1 = "<html>page1</html>"
+    html_p2 = "<html>page2</html>"
+
+    lot = ScrapedLot(
+        source="avito",
+        source_url="https://www.avito.ru/ekaterinburg/kvartiry/test",
+        source_id="LOT_1",
+        price_rub=5_000_000,
+    )
+
+    call_count = 0
+
+    async def mock_fetch_html(url: str, page: int) -> str:
+        return html_p1 if page == 1 else html_p2
+
+    def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
+        nonlocal call_count
+        call_count += 1
+        return [lot] if "page1" in html else []
+
+    with patch.object(scraper, "_fetch_serp_html", side_effect=mock_fetch_html):
+        with patch.object(scraper, "_parse_html", side_effect=mock_parse):
+            with patch.object(scraper, "sleep_between_requests", new=AsyncMock()):
+                result = await scraper.fetch_city_wide(pages=10, delay_override_sec=0)
+
+    assert len(result) == 1
+    assert result[0].source_id == "LOT_1"
+    # page1 parsed (returned lot) + page2 parsed (returned []) → stopped; page3-10 never called
+    assert call_count == 2
+
+
+# ---------------------------------------------------------------------------
+# 5. rotation returns False → immediate AvitoBlockedError
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_browser_mode_rotation_returns_false_raises_blocked() -> None:
+    """rotate_url set but _rotate_ip() returns False → must immediately raise AvitoBlockedError.
+
+    Covers the branch: firewall detected + rotate_url configured + _rotate_ip() == False
+    (e.g. rotation API timeout/failure) → we cannot recover → blocked.
+    """
+    scraper = AvitoScraper()
+    scraper._browser = _make_mock_browser(_FIREWALL_HTML)
+
+    with patch("app.services.scrapers.avito.settings") as mock_settings:
+        mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
+        mock_settings.avito_proxy_max_rotations = 2
+        with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=False)):
+            with pytest.raises(AvitoBlockedError, match="browser-mode"):
+                await scraper._fetch_serp_html("https://avito.ru/test", page=1)