"""Test anti-bot hardening: blocked exceptions + early abort.""" from unittest.mock import AsyncMock, MagicMock, call, patch import pytest from app.services.scrapers.avito_exceptions import ( AvitoBlockedError, AvitoRateLimitedError, ) # ── fetch_detail tests ────────────────────────────────────────── @pytest.mark.asyncio async def test_fetch_detail_raises_blocked_on_403() -> None: """fetch_detail должен raise AvitoBlockedError на HTTP 403.""" from app.services.scrapers.avito_detail import fetch_detail mock_session = AsyncMock() mock_response = MagicMock() mock_response.status_code = 403 mock_session.get = AsyncMock(return_value=mock_response) with pytest.raises(AvitoBlockedError, match="403"): await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=mock_session) @pytest.mark.asyncio async def test_fetch_detail_raises_rate_limited_on_429() -> None: """fetch_detail должен raise AvitoRateLimitedError на HTTP 429.""" from app.services.scrapers.avito_detail import fetch_detail mock_session = AsyncMock() mock_response = MagicMock() mock_response.status_code = 429 mock_session.get = AsyncMock(return_value=mock_response) with pytest.raises(AvitoRateLimitedError, match="429"): await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=mock_session) @pytest.mark.asyncio async def test_fetch_detail_raises_value_error_on_other_non_200() -> None: """fetch_detail должен raise ValueError на прочие non-200 (не 403/429).""" from app.services.scrapers.avito_detail import fetch_detail mock_session = AsyncMock() mock_response = MagicMock() mock_response.status_code = 503 mock_session.get = AsyncMock(return_value=mock_response) with pytest.raises(ValueError, match="503"): await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=mock_session) # ── SERP (avito.py) tests ─────────────────────────────────────── @pytest.mark.asyncio async def test_fetch_around_raises_blocked_on_403() -> None: """AvitoScraper.fetch_around должен raise AvitoBlockedError на HTTP 403.""" from app.services.scrapers.avito import AvitoScraper scraper = AvitoScraper() mock_session = AsyncMock() mock_response = MagicMock() mock_response.status_code = 403 mock_session.get = AsyncMock(return_value=mock_response) scraper._cffi = mock_session with pytest.raises(AvitoBlockedError): await scraper.fetch_around(56.84, 60.60, 1000, pages=1) @pytest.mark.asyncio async def test_fetch_around_raises_rate_limited_on_429() -> None: """AvitoScraper.fetch_around должен raise AvitoRateLimitedError на HTTP 429.""" from app.services.scrapers.avito import AvitoScraper scraper = AvitoScraper() mock_session = AsyncMock() mock_response = MagicMock() mock_response.status_code = 429 mock_session.get = AsyncMock(return_value=mock_response) scraper._cffi = mock_session with pytest.raises(AvitoRateLimitedError): await scraper.fetch_around(56.84, 60.60, 1000, pages=1) # ── Pipeline abort tests ──────────────────────────────────────── @pytest.mark.asyncio async def test_pipeline_aborts_after_3_consecutive_blocks() -> None: """Pipeline должен прервать sweep после 3 consecutive 403 (re-raise).""" from app.services.scrape_pipeline import run_avito_pipeline mock_db = MagicMock() mock_scraper = MagicMock() mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper) mock_scraper.__aexit__ = AsyncMock(return_value=None) mock_scraper.fetch_around = AsyncMock(return_value=[]) mock_scraper._cffi = None mock_row = MagicMock() mock_row.__getitem__ = lambda self, key: ( "https://www.avito.ru/ekaterinburg/kvartiry/test-1" if key == "source_url" else None ) mock_mappings = MagicMock() mock_mappings.all.return_value = [mock_row, mock_row, mock_row, mock_row] mock_execute = MagicMock() mock_execute.mappings.return_value = mock_mappings mock_db.execute.return_value = mock_execute block_error = AvitoBlockedError("HTTP 403") with ( patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper), patch( "app.services.scrape_pipeline.fetch_detail", AsyncMock(side_effect=block_error), ), patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()), patch( "curl_cffi.requests.AsyncSession", return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()), ), ): with pytest.raises(AvitoBlockedError): await run_avito_pipeline( mock_db, 56.8, 60.6, 1000, enrich_houses=False, enrich_detail_top_n=4, ) @pytest.mark.asyncio async def test_pipeline_sleep_between_detail_requests() -> None: """Pipeline должен вызывать asyncio.sleep между каждым detail request.""" from app.services.scrape_pipeline import run_avito_pipeline mock_db = MagicMock() mock_scraper = MagicMock() mock_scraper.__aenter__ = AsyncMock(return_value=mock_scraper) mock_scraper.__aexit__ = AsyncMock(return_value=None) mock_scraper.fetch_around = AsyncMock(return_value=[]) mock_scraper._cffi = None mock_row = MagicMock() mock_row.__getitem__ = lambda self, key: ( "https://www.avito.ru/ekaterinburg/kvartiry/test-1" if key == "source_url" else None ) mock_mappings = MagicMock() mock_mappings.all.return_value = [mock_row, mock_row, mock_row] mock_execute = MagicMock() mock_execute.mappings.return_value = mock_mappings mock_db.execute.return_value = mock_execute mock_enrichment = MagicMock() sleep_mock = AsyncMock() with ( patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper), patch( "app.services.scrape_pipeline.fetch_detail", AsyncMock(return_value=mock_enrichment), ), patch("app.services.scrape_pipeline.save_detail_enrichment", return_value=True), patch("app.services.scrape_pipeline.asyncio.sleep", sleep_mock), patch( "curl_cffi.requests.AsyncSession", return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()), ), patch("app.services.scrape_pipeline.random.uniform", return_value=1.0), ): await run_avito_pipeline( mock_db, 56.8, 60.6, 1000, enrich_houses=False, enrich_detail_top_n=3, request_delay_sec=7.0, ) assert sleep_mock.call_count == 2 for c in sleep_mock.call_args_list: assert c == call(7.0) @pytest.mark.asyncio async def test_city_sweep_marks_banned_on_block() -> None: """City sweep mark'ит run как banned при AvitoBlockedError. Migration 015 задокументировал 'banned' как 'Avito вернул 403/captcha' — именно этот сценарий. """ from app.services.scrape_pipeline import run_avito_city_sweep mock_db = MagicMock() mock_runs = MagicMock() mock_runs.is_cancelled.return_value = False mock_runs.update_heartbeat = MagicMock() mock_runs.mark_banned = MagicMock() mock_runs.mark_done = MagicMock() block_error = AvitoBlockedError("IP blocked at page=1") with ( patch("app.services.scrape_pipeline.scrape_runs", mock_runs), patch( "app.services.scrape_pipeline.run_avito_pipeline", AsyncMock(side_effect=block_error), ), patch( "curl_cffi.requests.AsyncSession", return_value=AsyncMock( __aenter__=AsyncMock(return_value=AsyncMock()), __aexit__=AsyncMock(return_value=None), ), ), ): result = await run_avito_city_sweep( mock_db, run_id=42, anchors=[(56.84, 60.60, "Центр"), (56.79, 60.53, "ЮЗ")], ) assert mock_runs.mark_banned.called assert not mock_runs.mark_done.called assert result.anchors_done == 1