Доступ ограничен: проблема с IP

"""Test anti-bot hardening: blocked exceptions + early abort.""" from unittest.mock import AsyncMock, MagicMock, call, patch import pytest from app.services.scrapers.avito_exceptions import ( AvitoBlockedError, AvitoContentBlockedError, AvitoRateLimitedError, ) # ── fetch_detail tests ────────────────────────────────────────── @pytest.mark.asyncio async def test_fetch_detail_raises_blocked_on_403() -> None: """fetch_detail должен raise AvitoBlockedError на HTTP 403.""" from app.services.scrapers.avito_detail import fetch_detail mock_session = AsyncMock() mock_response = MagicMock() mock_response.status_code = 403 mock_session.get = AsyncMock(return_value=mock_response) with pytest.raises(AvitoBlockedError, match="403"): await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=mock_session) @pytest.mark.asyncio async def test_fetch_detail_raises_rate_limited_on_429() -> None: """fetch_detail должен raise AvitoRateLimitedError на HTTP 429.""" from app.services.scrapers.avito_detail import fetch_detail mock_session = AsyncMock() mock_response = MagicMock() mock_response.status_code = 429 mock_session.get = AsyncMock(return_value=mock_response) with pytest.raises(AvitoRateLimitedError, match="429"): await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=mock_session) @pytest.mark.asyncio async def test_fetch_detail_raises_value_error_on_other_non_200() -> None: """fetch_detail должен raise ValueError на прочие non-200 (не 403/429).""" from app.services.scrapers.avito_detail import fetch_detail mock_session = AsyncMock() mock_response = MagicMock() mock_response.status_code = 503 mock_session.get = AsyncMock(return_value=mock_response) with pytest.raises(ValueError, match="503"): await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=mock_session) # ── SERP (avito.py) tests ─────────────────────────────────────── @pytest.mark.asyncio async def test_fetch_around_raises_content_blocked_on_page1_empty_html() -> None: """fetch_around должен raise AvitoContentBlockedError на HTTP 200 + 0 карточек на странице 1.""" from app.services.scrapers.avito import AvitoScraper scraper = AvitoScraper() mock_session = AsyncMock() mock_response = MagicMock() mock_response.status_code = 200 # Garbage HTML: нет data-marker="item" → _parse_html вернёт [] mock_response.text = "

captcha challenge

" mock_session.get = AsyncMock(return_value=mock_response) scraper._cffi = mock_session with pytest.raises(AvitoContentBlockedError, match="content-block"): await scraper.fetch_around(56.84, 60.60, 1000, pages=1) @pytest.mark.asyncio async def test_fetch_around_content_blocked_is_subtype_of_blocked() -> None: """AvitoContentBlockedError должен быть поймать как AvitoBlockedError (наследование).""" from app.services.scrapers.avito import AvitoScraper scraper = AvitoScraper() mock_session = AsyncMock() mock_response = MagicMock() mock_response.status_code = 200 mock_response.text = "" mock_session.get = AsyncMock(return_value=mock_response) scraper._cffi = mock_session with pytest.raises(AvitoBlockedError): await scraper.fetch_around(56.84, 60.60, 1000, pages=1) @pytest.mark.asyncio async def test_fetch_around_page_gt1_empty_returns_gracefully() -> None: """fetch_around на page>1 с 0 карточек должен тихо завершить пагинацию (не raise).""" from app.services.scrapers.avito import AvitoScraper scraper = AvitoScraper() mock_session = AsyncMock() # Страница 1: нормальный HTML с одной карточкой valid_html = ( "" '

' '' "2-к. квартира, 54,3 м², 5/9 эт." "" '' "

" "" ) # Страница 2: пустой HTML → end of pagination empty_html = "

no listings

" page1_resp = MagicMock() page1_resp.status_code = 200 page1_resp.text = valid_html page2_resp = MagicMock() page2_resp.status_code = 200 page2_resp.text = empty_html mock_session.get = AsyncMock(side_effect=[page1_resp, page2_resp]) scraper._cffi = mock_session with patch("app.services.scrapers.avito.AvitoScraper.sleep_between_requests", AsyncMock()): result = await scraper.fetch_around(56.84, 60.60, 1000, pages=2) # Должен вернуть лоты со страницы 1, не raise на странице 2 assert len(result) >= 1 @pytest.mark.asyncio async def test_fetch_around_raises_blocked_on_403() -> None: """AvitoScraper.fetch_around должен raise AvitoBlockedError на HTTP 403.""" from app.services.scrapers.avito import AvitoScraper scraper = AvitoScraper() mock_session = AsyncMock() mock_response = MagicMock() mock_response.status_code = 403 mock_session.get = AsyncMock(return_value=mock_response) scraper._cffi = mock_session with pytest.raises(AvitoBlockedError): await scraper.fetch_around(56.84, 60.60, 1000, pages=1) @pytest.mark.asyncio async def test_fetch_around_raises_rate_limited_on_429() -> None: """AvitoScraper.fetch_around должен raise AvitoRateLimitedError на HTTP 429.""" from app.services.scrapers.avito import AvitoScraper scraper = AvitoScraper() mock_session = AsyncMock() mock_response = MagicMock() mock_response.status_code = 429 mock_session.get = AsyncMock(return_value=mock_response) scraper._cffi = mock_session with pytest.raises(AvitoRateLimitedError): await scraper.fetch_around(56.84, 60.60, 1000, pages=1) # ── Pipeline abort tests ──────────────────────────────────────── @pytest.mark.asyncio async def test_pipeline_aborts_after_3_consecutive_blocks() -> None: """Pipeline должен прервать sweep после 3 consecutive 403 (re-raise).""" from app.services.scrape_pipeline import run_avito_pipeline mock_db = MagicMock() mock_scraper = MagicMock() mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper) mock_scraper.__aexit__ = AsyncMock(return_value=None) mock_scraper.fetch_around = AsyncMock(return_value=[]) mock_scraper._cffi = None mock_row = MagicMock() mock_row.__getitem__ = lambda self, key: ( "https://www.avito.ru/ekaterinburg/kvartiry/test-1" if key == "source_url" else None ) mock_mappings = MagicMock() mock_mappings.all.return_value = [mock_row, mock_row, mock_row, mock_row] mock_execute = MagicMock() mock_execute.mappings.return_value = mock_mappings mock_db.execute.return_value = mock_execute block_error = AvitoBlockedError("HTTP 403") with ( patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper), patch( "app.services.scrape_pipeline.fetch_detail", AsyncMock(side_effect=block_error), ), patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()), patch( "curl_cffi.requests.AsyncSession", return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()), ), ): with pytest.raises(AvitoBlockedError): await run_avito_pipeline( mock_db, 56.8, 60.6, 1000, enrich_houses=False, enrich_detail_top_n=4, ) @pytest.mark.asyncio async def test_pipeline_sleep_between_detail_requests() -> None: """Pipeline должен вызывать asyncio.sleep между каждым detail request.""" from app.services.scrape_pipeline import run_avito_pipeline mock_db = MagicMock() mock_scraper = MagicMock() mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper) mock_scraper.__aexit__ = AsyncMock(return_value=None) mock_scraper.fetch_around = AsyncMock(return_value=[]) mock_scraper._cffi = None mock_row = MagicMock() mock_row.__getitem__ = lambda self, key: ( "https://www.avito.ru/ekaterinburg/kvartiry/test-1" if key == "source_url" else None ) mock_mappings = MagicMock() mock_mappings.all.return_value = [mock_row, mock_row, mock_row] mock_execute = MagicMock() mock_execute.mappings.return_value = mock_mappings mock_db.execute.return_value = mock_execute mock_enrichment = MagicMock() sleep_mock = AsyncMock() with ( patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper), patch( "app.services.scrape_pipeline.fetch_detail", AsyncMock(return_value=mock_enrichment), ), patch("app.services.scrape_pipeline.save_detail_enrichment", return_value=True), patch("app.services.scrape_pipeline.asyncio.sleep", sleep_mock), patch( "curl_cffi.requests.AsyncSession", return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()), ), patch("app.services.scrape_pipeline.random.uniform", return_value=1.0), ): await run_avito_pipeline( mock_db, 56.8, 60.6, 1000, enrich_houses=False, enrich_detail_top_n=3, request_delay_sec=7.0, ) assert sleep_mock.call_count == 2 for c in sleep_mock.call_args_list: assert c == call(7.0) @pytest.mark.asyncio async def test_city_sweep_marks_banned_on_block() -> None: """City sweep mark'ит run как banned при AvitoBlockedError. Migration 015 задокументировал 'banned' как 'Avito вернул 403/captcha' — именно этот сценарий. """ from app.services.scrape_pipeline import run_avito_city_sweep mock_db = MagicMock() mock_runs = MagicMock() mock_runs.is_cancelled.return_value = False mock_runs.update_heartbeat = MagicMock() mock_runs.mark_banned = MagicMock() mock_runs.mark_done = MagicMock() block_error = AvitoBlockedError("IP blocked at page=1") with ( patch("app.services.scrape_pipeline.scrape_runs", mock_runs), patch( "app.services.scrape_pipeline.run_avito_pipeline", AsyncMock(side_effect=block_error), ), patch( "curl_cffi.requests.AsyncSession", return_value=AsyncMock( __aenter__=AsyncMock(return_value=AsyncMock()), __aexit__=AsyncMock(return_value=None), ), ), ): result = await run_avito_city_sweep( mock_db, run_id=42, anchors=[(56.84, 60.60, "Центр"), (56.79, 60.53, "ЮЗ")], ) assert mock_runs.mark_banned.called assert not mock_runs.mark_done.called assert result.anchors_done == 1 # ── #623 firewall-детект + ротация IP ──────────────────────────── _CARD_HTML = ( "" '

' '' "2-к. квартира, 54,3 м², 5/9 эт." "" '' "

" "" ) # Firewall-интерстициал Avito (HTTP 200, не 403) — IP-блок datacenter. _FIREWALL_HTML = ( "Доступ ограничен: проблема с IP" '

...

' ) @pytest.mark.asyncio async def test_fetch_around_raises_blocked_on_firewall_html_200() -> None: """HTTP 200 + firewall-страница «Доступ ограничен» → AvitoBlockedError (#623). Avito банит datacenter-IP отдачей firewall-интерстициала с кодом 200 (не 403) — раньше это молча проходило как «0 cards / content-block», теперь явный IP-блок. """ from app.services.scrapers.avito import AvitoScraper scraper = AvitoScraper() mock_session = AsyncMock() mock_response = MagicMock() mock_response.status_code = 200 mock_response.text = _FIREWALL_HTML mock_session.get = AsyncMock(return_value=mock_response) scraper._cffi = mock_session with pytest.raises(AvitoBlockedError, match="firewall"): await scraper.fetch_around(56.84, 60.60, 1000, pages=1) @pytest.mark.asyncio async def test_fetch_around_rotates_ip_then_succeeds(monkeypatch: pytest.MonkeyPatch) -> None: """При firewall + заданной changeip-ссылке: ротация IP и повтор страницы (#623).""" from app.services.scrapers import avito as avito_mod from app.services.scrapers.avito import AvitoScraper monkeypatch.setattr( avito_mod.settings, "avito_proxy_rotate_url", "https://changeip.example/?proxy_key=x" ) monkeypatch.setattr(avito_mod.settings, "avito_proxy_max_rotations", 1) scraper = AvitoScraper() fw = MagicMock() fw.status_code = 200 fw.text = _FIREWALL_HTML ok = MagicMock() ok.status_code = 200 ok.text = _CARD_HTML mock_session = AsyncMock() mock_session.get = AsyncMock(side_effect=[fw, ok]) scraper._cffi = mock_session with ( patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot, patch.object(AvitoScraper, "sleep_between_requests", AsyncMock()), ): result = await scraper.fetch_around(56.84, 60.60, 1000, pages=1) rot.assert_awaited_once() assert len(result) == 1 assert mock_session.get.await_count == 2 # firewall → rotate → retry OK @pytest.mark.asyncio async def test_fetch_around_blocked_when_rotation_exhausted( monkeypatch: pytest.MonkeyPatch, ) -> None: """Если ротация не сняла бан (max_rotations исчерпан) → AvitoBlockedError (#623).""" from app.services.scrapers import avito as avito_mod from app.services.scrapers.avito import AvitoScraper monkeypatch.setattr( avito_mod.settings, "avito_proxy_rotate_url", "https://changeip.example/?proxy_key=x" ) monkeypatch.setattr(avito_mod.settings, "avito_proxy_max_rotations", 1) scraper = AvitoScraper() fw = MagicMock() fw.status_code = 200 fw.text = _FIREWALL_HTML mock_session = AsyncMock() mock_session.get = AsyncMock(return_value=fw) # всегда firewall scraper._cffi = mock_session with patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot: with pytest.raises(AvitoBlockedError, match="firewall"): await scraper.fetch_around(56.84, 60.60, 1000, pages=1) assert rot.await_count == 1 # одна попытка ротации, потом сдаёмся # ── #623 sweep-pipeline proxy (SERP+detail через mobile proxy) ─── def test_avito_proxies_helper(monkeypatch: pytest.MonkeyPatch) -> None: """_avito_proxies: None без env, dict {http,https} при заданном AVITO_PROXY_URL. Регрессия: sweep создаёт shared-сессию, через неё идут и SERP, и detail — без прокси detail летит с datacenter-IP → 429 → run banned (#623 followup). """ from app.services import scrape_pipeline as sp monkeypatch.setattr(sp.settings, "avito_proxy_url", None) assert sp._avito_proxies() is None url = "http://u:p@mproxy.site:16928" monkeypatch.setattr(sp.settings, "avito_proxy_url", url) assert sp._avito_proxies() == {"http": url, "https": url}