{_FIREWALL_MARKERS[0]}

"""#901/#1805 — avito SERP fetch via BrowserFetcher (camoufox), curl_cffi fallback. Tests: 1. browser branch default (#1805): __aenter__ → _browser set AND _cffi fallback armed. 2. curl branch unchanged: _browser is None → _cffi session used directly. 3. firewall-detect in browser-mode: firewall HTML + no rotate URL + no cffi → AvitoBlockedError. 3b. #1805 curl_cffi fallback: browser firewall → curl retry returns HTML / re-raises if also banned. 4. pagination terminates via empty-parse: browser HTML with 0 lots → caller loop breaks. Без сети, без БД, без curl_cffi. """ from __future__ import annotations import os from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") from app.services.scrapers.avito import _FIREWALL_MARKERS, AvitoScraper from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError from app.services.scrapers.base import ScrapedLot # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- _NORMAL_HTML = "normal avito SERP" _FIREWALL_HTML = f"{_FIREWALL_MARKERS[0]}" def _sidecar_error(status: int, error: str) -> httpx.HTTPStatusError: """Build a tradein-browser sidecar HTTPStatusError with a real JSON body. Mirrors what BrowserFetcher.fetch re-raises: an httpx.HTTPStatusError whose `.response` is intact so `.response.json()["error"]` and `.response.status_code` work the same as in production. """ request = httpx.Request("POST", "http://tradein-browser:3000/fetch") response = httpx.Response(status, json={"error": error}, request=request) return httpx.HTTPStatusError(f"{status} error", request=request, response=response) def _make_mock_browser(html: str = _NORMAL_HTML) -> MagicMock: """Return a mock BrowserFetcher context manager that yields html from fetch().""" mock = MagicMock() mock.__aenter__ = AsyncMock(return_value=mock) mock.__aexit__ = AsyncMock(return_value=None) mock.fetch = AsyncMock(return_value=html) return mock # --------------------------------------------------------------------------- # 1. browser branch chosen # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_aenter_browser_mode_arms_cffi_fallback() -> None: """#1805: mode=browser → __aenter__ поднимает BrowserFetcher И curl_cffi-fallback. Раньше browser-mode оставлял _cffi=None; теперь curl_cffi-сессия строится тут же как fallback на firewall/пусто (см. _fetch_serp_html_browser). """ mock_browser = _make_mock_browser() scraper = AvitoScraper() with patch("app.services.scrapers.avito.settings") as mock_settings: mock_settings.scraper_fetch_mode = "browser" mock_settings.scraper_proxy_url = None mock_settings.avito_proxy_rotate_url = None mock_settings.avito_proxy_max_rotations = 2 with patch( "app.services.scrapers.avito.BrowserFetcher", return_value=mock_browser, ): result = await scraper.__aenter__() assert result is scraper assert scraper._browser is mock_browser # curl_cffi fallback armed (real AsyncSession built, no network/proxy). assert scraper._cffi is not None await scraper._cffi.close() @pytest.mark.asyncio async def test_aenter_proactive_rotate_when_rotate_url_set() -> None: """#1731: rotate_url configured → __aenter__ calls _rotate_ip exactly once at start. _rotate_ip is mocked (no real changeip HTTP / no 9s sleep). Browser mode used to keep BaseScraper setup out of the picture. """ mock_browser = _make_mock_browser() scraper = AvitoScraper() mock_rotate = AsyncMock(return_value=True) with patch("app.services.scrapers.avito.settings") as mock_settings: mock_settings.scraper_fetch_mode = "browser" mock_settings.scraper_proxy_url = None mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/changeip" mock_settings.avito_proxy_max_rotations = 4 with patch( "app.services.scrapers.avito.BrowserFetcher", return_value=mock_browser, ): with patch.object(scraper, "_rotate_ip", new=mock_rotate): result = await scraper.__aenter__() assert result is scraper mock_rotate.assert_awaited_once() @pytest.mark.asyncio async def test_aenter_no_proactive_rotate_when_rotate_url_none() -> None: """#1731: rotate_url is None → __aenter__ must NOT call _rotate_ip.""" mock_browser = _make_mock_browser() scraper = AvitoScraper() mock_rotate = AsyncMock(return_value=True) with patch("app.services.scrapers.avito.settings") as mock_settings: mock_settings.scraper_fetch_mode = "browser" mock_settings.scraper_proxy_url = None mock_settings.avito_proxy_rotate_url = None mock_settings.avito_proxy_max_rotations = 4 with patch( "app.services.scrapers.avito.BrowserFetcher", return_value=mock_browser, ): with patch.object(scraper, "_rotate_ip", new=mock_rotate): await scraper.__aenter__() mock_rotate.assert_not_awaited() @pytest.mark.asyncio async def test_aenter_proactive_rotate_failure_does_not_abort() -> None: """#1731: a failed proactive rotate (returns False / raises) must NOT abort __aenter__.""" mock_browser = _make_mock_browser() scraper = AvitoScraper() with patch("app.services.scrapers.avito.settings") as mock_settings: mock_settings.scraper_fetch_mode = "browser" mock_settings.scraper_proxy_url = None mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/changeip" mock_settings.avito_proxy_max_rotations = 4 with patch( "app.services.scrapers.avito.BrowserFetcher", return_value=mock_browser, ): # _rotate_ip raises — sweep must proceed (browser still attached) with patch.object( scraper, "_rotate_ip", new=AsyncMock(side_effect=RuntimeError("changeip down")) ): result = await scraper.__aenter__() assert result is scraper assert scraper._browser is mock_browser @pytest.mark.asyncio async def test_fetch_serp_html_browser_mode_returns_html() -> None: """_fetch_serp_html with _browser set returns HTML string, never touches _cffi.""" scraper = AvitoScraper() mock_browser = _make_mock_browser(_NORMAL_HTML) scraper._browser = mock_browser result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1) assert result == _NORMAL_HTML mock_browser.fetch.assert_awaited_once_with("https://www.avito.ru/test") # _cffi not armed (direct _browser assignment bypasses __aenter__, which would # otherwise build the curl_cffi fallback session); the browser path never touches it. assert scraper._cffi is None # --------------------------------------------------------------------------- # 2. curl branch unchanged # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_fetch_serp_html_curl_mode_uses_cffi() -> None: """Default mode (_browser is None) → _fetch_serp_html goes through curl path.""" scraper = AvitoScraper() # Mock the curl session returning HTTP 200 with normal HTML mock_response = MagicMock() mock_response.status_code = 200 mock_response.text = _NORMAL_HTML mock_cffi = MagicMock() mock_cffi.get = AsyncMock(return_value=mock_response) scraper._cffi = mock_cffi # _browser must be None (default curl path) assert scraper._browser is None result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1) assert result == _NORMAL_HTML mock_cffi.get.assert_awaited_once_with("https://www.avito.ru/test") # --------------------------------------------------------------------------- # 3. firewall-detect in browser-mode → AvitoBlockedError (no rotate URL) # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_browser_mode_firewall_raises_blocked_error() -> None: """Browser firewall + no rotate URL + no curl fallback (_cffi=None) → AvitoBlockedError.""" scraper = AvitoScraper() mock_browser = _make_mock_browser(_FIREWALL_HTML) scraper._browser = mock_browser scraper._cffi = None # no curl_cffi fallback available (pipeline shared-browser path) with patch("app.services.scrapers.avito.settings") as mock_settings: mock_settings.avito_proxy_rotate_url = None # no rotation configured mock_settings.avito_proxy_max_rotations = 2 with pytest.raises(AvitoBlockedError, match="browser-mode"): await scraper._fetch_serp_html("https://www.avito.ru/test", page=1) # --------------------------------------------------------------------------- # 3b. #1805 — curl_cffi fallback when browser returns firewall # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_browser_firewall_falls_back_to_cffi_returns_html() -> None: """#1805: browser firewall + curl_cffi fallback armed → curl returns clean HTML.""" scraper = AvitoScraper() scraper._browser = _make_mock_browser(_FIREWALL_HTML) # curl_cffi fallback: HTTP 200 with clean (non-firewall) HTML. mock_resp = MagicMock() mock_resp.status_code = 200 mock_resp.text = _NORMAL_HTML mock_cffi = MagicMock() mock_cffi.get = AsyncMock(return_value=mock_resp) scraper._cffi = mock_cffi with patch("app.services.scrapers.avito.settings") as mock_settings: mock_settings.avito_proxy_rotate_url = None mock_settings.avito_proxy_max_rotations = 0 mock_settings.scraper_proxy_url = None result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1) assert result == _NORMAL_HTML mock_cffi.get.assert_awaited_once_with("https://www.avito.ru/test") @pytest.mark.asyncio async def test_browser_firewall_cffi_fallback_also_firewall_raises() -> None: """#1805: browser firewall AND curl_cffi fallback also firewall → AvitoBlockedError.""" scraper = AvitoScraper() scraper._browser = _make_mock_browser(_FIREWALL_HTML) # curl_cffi fallback: HTTP 200 but ALSO firewall HTML → no escape. mock_resp = MagicMock() mock_resp.status_code = 200 mock_resp.text = _FIREWALL_HTML mock_cffi = MagicMock() mock_cffi.get = AsyncMock(return_value=mock_resp) scraper._cffi = mock_cffi with patch("app.services.scrapers.avito.settings") as mock_settings: mock_settings.avito_proxy_rotate_url = None mock_settings.avito_proxy_max_rotations = 0 mock_settings.scraper_proxy_url = None # no backconnect reconnect retries # curl fallback raises its own AvitoBlockedError (firewall на HTTP 200); # browser branch re-raises it as a hard-block. with pytest.raises(AvitoBlockedError): await scraper._fetch_serp_html("https://www.avito.ru/test", page=1) @pytest.mark.asyncio async def test_browser_mode_firewall_rotates_and_retries() -> None: """Browser returns firewall on first call, normal HTML after rotation → returns HTML.""" scraper = AvitoScraper() # First call → firewall; second call → normal HTML mock_browser = MagicMock() mock_browser.fetch = AsyncMock(side_effect=[_FIREWALL_HTML, _NORMAL_HTML]) scraper._browser = mock_browser with patch("app.services.scrapers.avito.settings") as mock_settings: mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/rotate" mock_settings.avito_proxy_max_rotations = 1 with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=True)): result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=2) assert result == _NORMAL_HTML assert mock_browser.fetch.await_count == 2 # --------------------------------------------------------------------------- # 4. pagination terminates via empty-parse # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_fetch_city_wide_browser_mode_stops_on_empty_parse() -> None: """fetch_city_wide with browser mode: page=1 returns lots, page=2 returns [] → stops. Verifies the documented invariant: callers break on `not lots` regardless of fetch mode. _fetch_serp_html is mocked at method level (browser vs curl is already covered by unit tests above); here we confirm the caller loop behaviour. """ scraper = AvitoScraper() html_p1 = "page1" html_p2 = "page2" lot = ScrapedLot( source="avito", source_url="https://www.avito.ru/ekaterinburg/kvartiry/test", source_id="LOT_1", price_rub=5_000_000, ) call_count = 0 async def mock_fetch_html(url: str, page: int) -> str: return html_p1 if page == 1 else html_p2 def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]: nonlocal call_count call_count += 1 return [lot] if "page1" in html else [] with patch.object(scraper, "_fetch_serp_html", side_effect=mock_fetch_html): with patch.object(scraper, "_parse_html", side_effect=mock_parse): with patch.object(scraper, "sleep_between_requests", new=AsyncMock()): result = await scraper.fetch_city_wide(pages=10, delay_override_sec=0) assert len(result) == 1 assert result[0].source_id == "LOT_1" # page1 parsed (returned lot) + page2 parsed (returned []) → stopped; page3-10 never called assert call_count == 2 # --------------------------------------------------------------------------- # 5. rotation returns False → immediate AvitoBlockedError # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_browser_mode_rotation_returns_false_raises_blocked() -> None: """rotate_url set but _rotate_ip() returns False → must immediately raise AvitoBlockedError. Covers the branch: firewall detected + rotate_url configured + _rotate_ip() == False (e.g. rotation API timeout/failure) → we cannot recover → blocked. """ scraper = AvitoScraper() scraper._browser = _make_mock_browser(_FIREWALL_HTML) with patch("app.services.scrapers.avito.settings") as mock_settings: mock_settings.avito_proxy_rotate_url = "http://proxy/rotate" mock_settings.avito_proxy_max_rotations = 2 with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=False)): with pytest.raises(AvitoBlockedError, match="browser-mode"): await scraper._fetch_serp_html("https://avito.ru/test", page=1) # --------------------------------------------------------------------------- # 6. #356 — sidecar HTTPStatusError (500/503) → graceful AvitoRateLimitedError # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_sidecar_proxy_too_many_requests_rotates_then_succeeds() -> None: """#356: sidecar 500 NS_ERROR_PROXY_TOO_MANY_REQUESTS, rotation available → after IP rotation a retry succeeds and the clean HTML is returned.""" scraper = AvitoScraper() mock_browser = MagicMock() # First fetch raises the soft-ban 500; after rotation the second fetch is clean. mock_browser.fetch = AsyncMock( side_effect=[ _sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"), _NORMAL_HTML, ] ) scraper._browser = mock_browser mock_rotate = AsyncMock(return_value=True) with patch("app.services.scrapers.avito.settings") as mock_settings: mock_settings.avito_proxy_rotate_url = "http://proxy/rotate" mock_settings.avito_proxy_max_rotations = 2 with patch.object(scraper, "_rotate_ip", new=mock_rotate): result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=3) assert result == _NORMAL_HTML mock_rotate.assert_awaited_once() assert mock_browser.fetch.await_count == 2 @pytest.mark.asyncio async def test_sidecar_proxy_too_many_requests_exhausted_raises_rate_limited() -> None: """#356: sidecar 500 proxy-too-many with rotation budget exhausted → AvitoRateLimitedError (NOT a bare HTTPStatusError, NOT AvitoBlockedError).""" scraper = AvitoScraper() mock_browser = MagicMock() mock_browser.fetch = AsyncMock( side_effect=_sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS") ) scraper._browser = mock_browser mock_rotate = AsyncMock(return_value=True) with patch("app.services.scrapers.avito.settings") as mock_settings: mock_settings.avito_proxy_rotate_url = "http://proxy/rotate" mock_settings.avito_proxy_max_rotations = 2 with patch.object(scraper, "_rotate_ip", new=mock_rotate): with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"): await scraper._fetch_serp_html("https://www.avito.ru/test", page=3) # 2 rotations consumed + 1 final fetch that exhausts the budget. assert mock_rotate.await_count == 2 assert mock_browser.fetch.await_count == 3 @pytest.mark.asyncio async def test_sidecar_transient_timeout_exhausts_budget_no_infinite_loop() -> None: """#356: sidecar 500 transient (Timeout) repeated past the transient budget with NO rotations → AvitoRateLimitedError after bounded retries (no infinite loop).""" scraper = AvitoScraper() mock_browser = MagicMock() mock_browser.fetch = AsyncMock( side_effect=_sidecar_error(500, "TimeoutError: Page.goto: Timeout 60000ms exceeded") ) scraper._browser = mock_browser mock_rotate = AsyncMock(return_value=True) with patch("app.services.scrapers.avito.settings") as mock_settings: mock_settings.avito_proxy_rotate_url = None # no rotation → transient budget only mock_settings.avito_proxy_max_rotations = 0 # Patch sleep so the bounded backoff doesn't slow the test. with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): with patch.object(scraper, "_rotate_ip", new=mock_rotate): with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"): await scraper._fetch_serp_html("https://www.avito.ru/test", page=5) # Transient is NOT a soft-ban → never rotates. mock_rotate.assert_not_awaited() # Bounded: _AVITO_SIDECAR_TRANSIENT_RETRIES (2) retries + final attempt = 3. assert mock_browser.fetch.await_count == 3 @pytest.mark.asyncio async def test_sidecar_503_browser_unavailable_exhausted_raises_rate_limited() -> None: """#356: sidecar 503 'browser unavailable (proxy may be down)' with rotation exhausted → AvitoRateLimitedError (soft-ban classification via status==503).""" scraper = AvitoScraper() mock_browser = MagicMock() mock_browser.fetch = AsyncMock( side_effect=_sidecar_error(503, "browser unavailable (proxy may be down)") ) scraper._browser = mock_browser mock_rotate = AsyncMock(return_value=True) with patch("app.services.scrapers.avito.settings") as mock_settings: mock_settings.avito_proxy_rotate_url = "http://proxy/rotate" mock_settings.avito_proxy_max_rotations = 1 with patch.object(scraper, "_rotate_ip", new=mock_rotate): with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"): await scraper._fetch_serp_html("https://www.avito.ru/test", page=7) # 1 rotation consumed then exhausted. assert mock_rotate.await_count == 1 assert mock_browser.fetch.await_count == 2 @pytest.mark.asyncio async def test_sidecar_transient_then_soft_ban_independent_budgets() -> None: """#356: смешанная последовательность transient → soft-ban пинит, что бюджеты независимы и корректно чередуются в одном while-цикле. attempt 1: transient Timeout → тратит transient_left (1 из 2), backoff, continue. attempt 2: too_many_requests → soft-ban, тратит ротацию (1 из 1), continue. attempt 3: too_many_requests, ротация исчерпана → AvitoRateLimitedError. """ scraper = AvitoScraper() mock_browser = MagicMock() mock_browser.fetch = AsyncMock( side_effect=[ _sidecar_error(500, "TimeoutError: Page.goto: Timeout 60000ms exceeded"), _sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"), _sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"), ] ) scraper._browser = mock_browser mock_rotate = AsyncMock(return_value=True) with patch("app.services.scrapers.avito.settings") as mock_settings: mock_settings.avito_proxy_rotate_url = "http://proxy/rotate" mock_settings.avito_proxy_max_rotations = 1 with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): with patch.object(scraper, "_rotate_ip", new=mock_rotate): with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"): await scraper._fetch_serp_html("https://www.avito.ru/test", page=9) # 1 transient retry + 1 rotation, оба бюджета тронуты до исчерпания. assert mock_rotate.await_count == 1 assert mock_browser.fetch.await_count == 3