diff --git a/tradein-mvp/backend/app/services/scrapers/avito.py b/tradein-mvp/backend/app/services/scrapers/avito.py index 88343564..ae49f725 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito.py +++ b/tradein-mvp/backend/app/services/scrapers/avito.py @@ -41,6 +41,7 @@ from app.services.scrapers.avito_exceptions import ( AvitoRateLimitedError, ) from app.services.scrapers.base import BaseScraper, ScrapedLot +from app.services.scrapers.browser_fetcher import BrowserFetcher logger = logging.getLogger(__name__) @@ -215,10 +216,16 @@ class AvitoScraper(BaseScraper): super().__init__() self.request_delay_sec = get_scraper_delay(self.name) self._cffi: AsyncSession | None = None + self._browser: BrowserFetcher | None = None # #823: счётчик карточек, которые не удалось распарсить из-за неожиданной структуры DOM. self.parse_failures: int = 0 async def __aenter__(self) -> AvitoScraper: + if settings.scraper_fetch_mode == "browser": + self._browser = BrowserFetcher() + await self._browser.__aenter__() + logger.info("avito: SERP fetch via BrowserFetcher (camoufox) — #901") + return self await super().__aenter__() proxies = None if settings.scraper_proxy_url: @@ -242,6 +249,8 @@ class AvitoScraper(BaseScraper): return self async def __aexit__(self, *args: Any) -> None: + if self._browser is not None: + await self._browser.__aexit__(*args) if self._cffi is not None: await self._cffi.close() await super().__aexit__(*args) @@ -277,9 +286,31 @@ class AvitoScraper(BaseScraper): Returns: HTML при HTTP 200, либо ``None`` при non-200 (конец пагинации). + В browser-mode HTTP-статус недоступен: метод возвращает HTML или + поднимает ``AvitoBlockedError`` (pagination ends via empty-parse in callers). Raises: AvitoBlockedError / AvitoRateLimitedError — если бан не снят ротацией. """ + if self._browser is not None: + # Browser-mode (#901): no HTTP status code — soft firewall-detect + rotate. + max_rot = settings.avito_proxy_max_rotations if settings.avito_proxy_rotate_url else 0 + rot_done = 0 + while True: + html = await self._browser.fetch(url) + if _is_firewall_page(html): + if rot_done < max_rot and await self._rotate_ip(): + rot_done += 1 + logger.info( + "avito page=%d browser firewall — retry after rotation #%d", + page, + rot_done, + ) + continue + logger.error("avito SERP firewall in browser-mode page=%d url=%s", page, url) + raise AvitoBlockedError( + f"Avito SERP firewall (browser-mode) at page={page} — IP banned" + ) + return html assert self._cffi is not None max_rot = settings.avito_proxy_max_rotations if settings.avito_proxy_rotate_url else 0 rot_done = 0 diff --git a/tradein-mvp/backend/tests/test_avito_browser_mode.py b/tradein-mvp/backend/tests/test_avito_browser_mode.py new file mode 100644 index 00000000..f6be8369 --- /dev/null +++ b/tradein-mvp/backend/tests/test_avito_browser_mode.py @@ -0,0 +1,225 @@ +"""#901 — avito SERP fetch via BrowserFetcher behind scraper_fetch_mode flag. + +Tests: +1. browser branch chosen: mode="browser" → _browser set, _cffi is None, HTML from mock browser. +2. curl branch unchanged: default mode → _browser is None, _cffi session used. +3. firewall-detect in browser-mode: firewall HTML + no rotate URL → AvitoBlockedError. +4. pagination terminates via empty-parse: browser HTML with 0 lots → caller loop breaks. + +Без сети, без БД, без curl_cffi. +""" + +from __future__ import annotations + +import os +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") + +from app.services.scrapers.avito import _FIREWALL_MARKERS, AvitoScraper +from app.services.scrapers.avito_exceptions import AvitoBlockedError +from app.services.scrapers.base import ScrapedLot + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_NORMAL_HTML = "normal avito SERP" +_FIREWALL_HTML = f"{_FIREWALL_MARKERS[0]}" + + +def _make_mock_browser(html: str = _NORMAL_HTML) -> MagicMock: + """Return a mock BrowserFetcher context manager that yields html from fetch().""" + mock = MagicMock() + mock.__aenter__ = AsyncMock(return_value=mock) + mock.__aexit__ = AsyncMock(return_value=None) + mock.fetch = AsyncMock(return_value=html) + return mock + + +# --------------------------------------------------------------------------- +# 1. browser branch chosen +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_aenter_browser_mode_sets_browser_attribute() -> None: + """mode=browser → __aenter__ instantiates BrowserFetcher, _cffi stays None.""" + mock_browser = _make_mock_browser() + scraper = AvitoScraper() + + with patch("app.services.scrapers.avito.settings") as mock_settings: + mock_settings.scraper_fetch_mode = "browser" + mock_settings.scraper_proxy_url = None + mock_settings.avito_proxy_rotate_url = None + mock_settings.avito_proxy_max_rotations = 2 + + with patch( + "app.services.scrapers.avito.BrowserFetcher", + return_value=mock_browser, + ): + # Patch BaseScraper.__aenter__ so we skip its real setup (DB/network) + with patch( + "app.services.scrapers.base.BaseScraper.__aenter__", + new=AsyncMock(return_value=scraper), + ): + result = await scraper.__aenter__() + + assert result is scraper + assert scraper._browser is mock_browser + assert scraper._cffi is None + + +@pytest.mark.asyncio +async def test_fetch_serp_html_browser_mode_returns_html() -> None: + """_fetch_serp_html with _browser set returns HTML string, never touches _cffi.""" + scraper = AvitoScraper() + mock_browser = _make_mock_browser(_NORMAL_HTML) + scraper._browser = mock_browser + + result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1) + + assert result == _NORMAL_HTML + mock_browser.fetch.assert_awaited_once_with("https://www.avito.ru/test") + # _cffi was never set — assert remains None + assert scraper._cffi is None + + +# --------------------------------------------------------------------------- +# 2. curl branch unchanged +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_fetch_serp_html_curl_mode_uses_cffi() -> None: + """Default mode (_browser is None) → _fetch_serp_html goes through curl path.""" + scraper = AvitoScraper() + + # Mock the curl session returning HTTP 200 with normal HTML + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = _NORMAL_HTML + + mock_cffi = MagicMock() + mock_cffi.get = AsyncMock(return_value=mock_response) + scraper._cffi = mock_cffi + + # _browser must be None (default curl path) + assert scraper._browser is None + + result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1) + + assert result == _NORMAL_HTML + mock_cffi.get.assert_awaited_once_with("https://www.avito.ru/test") + + +# --------------------------------------------------------------------------- +# 3. firewall-detect in browser-mode → AvitoBlockedError (no rotate URL) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_browser_mode_firewall_raises_blocked_error() -> None: + """Browser returns firewall page + no rotate URL configured → AvitoBlockedError.""" + scraper = AvitoScraper() + mock_browser = _make_mock_browser(_FIREWALL_HTML) + scraper._browser = mock_browser + + with patch("app.services.scrapers.avito.settings") as mock_settings: + mock_settings.avito_proxy_rotate_url = None # no rotation configured + mock_settings.avito_proxy_max_rotations = 2 + + with pytest.raises(AvitoBlockedError, match="browser-mode"): + await scraper._fetch_serp_html("https://www.avito.ru/test", page=1) + + +@pytest.mark.asyncio +async def test_browser_mode_firewall_rotates_and_retries() -> None: + """Browser returns firewall on first call, normal HTML after rotation → returns HTML.""" + scraper = AvitoScraper() + + # First call → firewall; second call → normal HTML + mock_browser = MagicMock() + mock_browser.fetch = AsyncMock(side_effect=[_FIREWALL_HTML, _NORMAL_HTML]) + scraper._browser = mock_browser + + with patch("app.services.scrapers.avito.settings") as mock_settings: + mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/rotate" + mock_settings.avito_proxy_max_rotations = 1 + + with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=True)): + result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=2) + + assert result == _NORMAL_HTML + assert mock_browser.fetch.await_count == 2 + + +# --------------------------------------------------------------------------- +# 4. pagination terminates via empty-parse +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_fetch_city_wide_browser_mode_stops_on_empty_parse() -> None: + """fetch_city_wide with browser mode: page=1 returns lots, page=2 returns [] → stops. + + Verifies the documented invariant: callers break on `not lots` regardless of + fetch mode. _fetch_serp_html is mocked at method level (browser vs curl is + already covered by unit tests above); here we confirm the caller loop behaviour. + """ + scraper = AvitoScraper() + + html_p1 = "page1" + html_p2 = "page2" + + lot = ScrapedLot( + source="avito", + source_url="https://www.avito.ru/ekaterinburg/kvartiry/test", + source_id="LOT_1", + price_rub=5_000_000, + ) + + call_count = 0 + + async def mock_fetch_html(url: str, page: int) -> str: + return html_p1 if page == 1 else html_p2 + + def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]: + nonlocal call_count + call_count += 1 + return [lot] if "page1" in html else [] + + with patch.object(scraper, "_fetch_serp_html", side_effect=mock_fetch_html): + with patch.object(scraper, "_parse_html", side_effect=mock_parse): + with patch.object(scraper, "sleep_between_requests", new=AsyncMock()): + result = await scraper.fetch_city_wide(pages=10, delay_override_sec=0) + + assert len(result) == 1 + assert result[0].source_id == "LOT_1" + # page1 parsed (returned lot) + page2 parsed (returned []) → stopped; page3-10 never called + assert call_count == 2 + + +# --------------------------------------------------------------------------- +# 5. rotation returns False → immediate AvitoBlockedError +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_browser_mode_rotation_returns_false_raises_blocked() -> None: + """rotate_url set but _rotate_ip() returns False → must immediately raise AvitoBlockedError. + + Covers the branch: firewall detected + rotate_url configured + _rotate_ip() == False + (e.g. rotation API timeout/failure) → we cannot recover → blocked. + """ + scraper = AvitoScraper() + scraper._browser = _make_mock_browser(_FIREWALL_HTML) + + with patch("app.services.scrapers.avito.settings") as mock_settings: + mock_settings.avito_proxy_rotate_url = "http://proxy/rotate" + mock_settings.avito_proxy_max_rotations = 2 + with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=False)): + with pytest.raises(AvitoBlockedError, match="browser-mode"): + await scraper._fetch_serp_html("https://avito.ru/test", page=1)