{_FIREWALL_MARKERS[0]}

"""#901 — avito SERP fetch via BrowserFetcher behind scraper_fetch_mode flag. Tests: 1. browser branch chosen: mode="browser" → _browser set, _cffi is None, HTML from mock browser. 2. curl branch unchanged: default mode → _browser is None, _cffi session used. 3. firewall-detect in browser-mode: firewall HTML + no rotate URL → AvitoBlockedError. 4. pagination terminates via empty-parse: browser HTML with 0 lots → caller loop breaks. Без сети, без БД, без curl_cffi. """ from __future__ import annotations import os from unittest.mock import AsyncMock, MagicMock, patch import pytest os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") from app.services.scrapers.avito import _FIREWALL_MARKERS, AvitoScraper from app.services.scrapers.avito_exceptions import AvitoBlockedError from app.services.scrapers.base import ScrapedLot # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- _NORMAL_HTML = "normal avito SERP" _FIREWALL_HTML = f"{_FIREWALL_MARKERS[0]}" def _make_mock_browser(html: str = _NORMAL_HTML) -> MagicMock: """Return a mock BrowserFetcher context manager that yields html from fetch().""" mock = MagicMock() mock.__aenter__ = AsyncMock(return_value=mock) mock.__aexit__ = AsyncMock(return_value=None) mock.fetch = AsyncMock(return_value=html) return mock # --------------------------------------------------------------------------- # 1. browser branch chosen # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_aenter_browser_mode_sets_browser_attribute() -> None: """mode=browser → __aenter__ instantiates BrowserFetcher, _cffi stays None.""" mock_browser = _make_mock_browser() scraper = AvitoScraper() with patch("app.services.scrapers.avito.settings") as mock_settings: mock_settings.scraper_fetch_mode = "browser" mock_settings.scraper_proxy_url = None mock_settings.avito_proxy_rotate_url = None mock_settings.avito_proxy_max_rotations = 2 with patch( "app.services.scrapers.avito.BrowserFetcher", return_value=mock_browser, ): # Patch BaseScraper.__aenter__ so we skip its real setup (DB/network) with patch( "app.services.scrapers.base.BaseScraper.__aenter__", new=AsyncMock(return_value=scraper), ): result = await scraper.__aenter__() assert result is scraper assert scraper._browser is mock_browser assert scraper._cffi is None @pytest.mark.asyncio async def test_fetch_serp_html_browser_mode_returns_html() -> None: """_fetch_serp_html with _browser set returns HTML string, never touches _cffi.""" scraper = AvitoScraper() mock_browser = _make_mock_browser(_NORMAL_HTML) scraper._browser = mock_browser result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1) assert result == _NORMAL_HTML mock_browser.fetch.assert_awaited_once_with("https://www.avito.ru/test") # _cffi was never set — assert remains None assert scraper._cffi is None # --------------------------------------------------------------------------- # 2. curl branch unchanged # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_fetch_serp_html_curl_mode_uses_cffi() -> None: """Default mode (_browser is None) → _fetch_serp_html goes through curl path.""" scraper = AvitoScraper() # Mock the curl session returning HTTP 200 with normal HTML mock_response = MagicMock() mock_response.status_code = 200 mock_response.text = _NORMAL_HTML mock_cffi = MagicMock() mock_cffi.get = AsyncMock(return_value=mock_response) scraper._cffi = mock_cffi # _browser must be None (default curl path) assert scraper._browser is None result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1) assert result == _NORMAL_HTML mock_cffi.get.assert_awaited_once_with("https://www.avito.ru/test") # --------------------------------------------------------------------------- # 3. firewall-detect in browser-mode → AvitoBlockedError (no rotate URL) # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_browser_mode_firewall_raises_blocked_error() -> None: """Browser returns firewall page + no rotate URL configured → AvitoBlockedError.""" scraper = AvitoScraper() mock_browser = _make_mock_browser(_FIREWALL_HTML) scraper._browser = mock_browser with patch("app.services.scrapers.avito.settings") as mock_settings: mock_settings.avito_proxy_rotate_url = None # no rotation configured mock_settings.avito_proxy_max_rotations = 2 with pytest.raises(AvitoBlockedError, match="browser-mode"): await scraper._fetch_serp_html("https://www.avito.ru/test", page=1) @pytest.mark.asyncio async def test_browser_mode_firewall_rotates_and_retries() -> None: """Browser returns firewall on first call, normal HTML after rotation → returns HTML.""" scraper = AvitoScraper() # First call → firewall; second call → normal HTML mock_browser = MagicMock() mock_browser.fetch = AsyncMock(side_effect=[_FIREWALL_HTML, _NORMAL_HTML]) scraper._browser = mock_browser with patch("app.services.scrapers.avito.settings") as mock_settings: mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/rotate" mock_settings.avito_proxy_max_rotations = 1 with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=True)): result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=2) assert result == _NORMAL_HTML assert mock_browser.fetch.await_count == 2 # --------------------------------------------------------------------------- # 4. pagination terminates via empty-parse # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_fetch_city_wide_browser_mode_stops_on_empty_parse() -> None: """fetch_city_wide with browser mode: page=1 returns lots, page=2 returns [] → stops. Verifies the documented invariant: callers break on `not lots` regardless of fetch mode. _fetch_serp_html is mocked at method level (browser vs curl is already covered by unit tests above); here we confirm the caller loop behaviour. """ scraper = AvitoScraper() html_p1 = "page1" html_p2 = "page2" lot = ScrapedLot( source="avito", source_url="https://www.avito.ru/ekaterinburg/kvartiry/test", source_id="LOT_1", price_rub=5_000_000, ) call_count = 0 async def mock_fetch_html(url: str, page: int) -> str: return html_p1 if page == 1 else html_p2 def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]: nonlocal call_count call_count += 1 return [lot] if "page1" in html else [] with patch.object(scraper, "_fetch_serp_html", side_effect=mock_fetch_html): with patch.object(scraper, "_parse_html", side_effect=mock_parse): with patch.object(scraper, "sleep_between_requests", new=AsyncMock()): result = await scraper.fetch_city_wide(pages=10, delay_override_sec=0) assert len(result) == 1 assert result[0].source_id == "LOT_1" # page1 parsed (returned lot) + page2 parsed (returned []) → stopped; page3-10 never called assert call_count == 2 # --------------------------------------------------------------------------- # 5. rotation returns False → immediate AvitoBlockedError # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_browser_mode_rotation_returns_false_raises_blocked() -> None: """rotate_url set but _rotate_ip() returns False → must immediately raise AvitoBlockedError. Covers the branch: firewall detected + rotate_url configured + _rotate_ip() == False (e.g. rotation API timeout/failure) → we cannot recover → blocked. """ scraper = AvitoScraper() scraper._browser = _make_mock_browser(_FIREWALL_HTML) with patch("app.services.scrapers.avito.settings") as mock_settings: mock_settings.avito_proxy_rotate_url = "http://proxy/rotate" mock_settings.avito_proxy_max_rotations = 2 with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=False)): with pytest.raises(AvitoBlockedError, match="browser-mode"): await scraper._fetch_serp_html("https://avito.ru/test", page=1)