diff --git a/tradein-mvp/backend/app/services/scrapers/avito.py b/tradein-mvp/backend/app/services/scrapers/avito.py index 6aaa8c59..0449118d 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito.py +++ b/tradein-mvp/backend/app/services/scrapers/avito.py @@ -25,7 +25,11 @@ from curl_cffi.requests import AsyncSession from selectolax.parser import HTMLParser from app.services.scraper_settings import get_scraper_delay -from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError +from app.services.scrapers.avito_exceptions import ( + AvitoBlockedError, + AvitoContentBlockedError, + AvitoRateLimitedError, +) from app.services.scrapers.base import BaseScraper, ScrapedLot logger = logging.getLogger(__name__) @@ -261,6 +265,15 @@ class AvitoScraper(BaseScraper): lots = self._parse_html(response.text, source_url_base=url) if not lots: + if page == 1: + logger.error( + "avito SERP page=1 returned HTTP 200 but 0 cards — " + "likely content-block/captcha url=%s", + url, + ) + raise AvitoContentBlockedError( + "Avito SERP HTTP 200 with 0 cards on page=1 — content-block suspected" + ) logger.info("avito page=%d: 0 lots — end of pagination", page) break all_lots.extend(lots) diff --git a/tradein-mvp/backend/app/services/scrapers/avito_exceptions.py b/tradein-mvp/backend/app/services/scrapers/avito_exceptions.py index 0ea56350..b2ea0103 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito_exceptions.py +++ b/tradein-mvp/backend/app/services/scrapers/avito_exceptions.py @@ -13,5 +13,13 @@ class AvitoRateLimitedError(AvitoError): """HTTP 429 от Avito — rate limit triggered.""" +class AvitoContentBlockedError(AvitoBlockedError): + """HTTP 200 от Avito, но 0 карточек на первой странице — content-block / captcha. + + Наследует AvitoBlockedError, чтобы существующие except-блоки на AvitoBlockedError + продолжали ловить этот сигнал (mark_banned, pipeline abort и т.д.). + """ + + class AvitoParseError(AvitoError): """Cannot parse Avito HTML structure (selector changes).""" diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py b/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py index 0e64073d..c18b2274 100644 --- a/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py +++ b/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py @@ -6,6 +6,7 @@ import pytest from app.services.scrapers.avito_exceptions import ( AvitoBlockedError, + AvitoContentBlockedError, AvitoRateLimitedError, ) @@ -57,6 +58,81 @@ async def test_fetch_detail_raises_value_error_on_other_non_200() -> None: # ── SERP (avito.py) tests ─────────────────────────────────────── +@pytest.mark.asyncio +async def test_fetch_around_raises_content_blocked_on_page1_empty_html() -> None: + """fetch_around должен raise AvitoContentBlockedError на HTTP 200 + 0 карточек на странице 1.""" + from app.services.scrapers.avito import AvitoScraper + + scraper = AvitoScraper() + mock_session = AsyncMock() + mock_response = MagicMock() + mock_response.status_code = 200 + # Garbage HTML: нет data-marker="item" → _parse_html вернёт [] + mock_response.text = "

captcha challenge

" + mock_session.get = AsyncMock(return_value=mock_response) + scraper._cffi = mock_session + + with pytest.raises(AvitoContentBlockedError, match="content-block"): + await scraper.fetch_around(56.84, 60.60, 1000, pages=1) + + +@pytest.mark.asyncio +async def test_fetch_around_content_blocked_is_subtype_of_blocked() -> None: + """AvitoContentBlockedError должен быть поймать как AvitoBlockedError (наследование).""" + from app.services.scrapers.avito import AvitoScraper + + scraper = AvitoScraper() + mock_session = AsyncMock() + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = "" + mock_session.get = AsyncMock(return_value=mock_response) + scraper._cffi = mock_session + + with pytest.raises(AvitoBlockedError): + await scraper.fetch_around(56.84, 60.60, 1000, pages=1) + + +@pytest.mark.asyncio +async def test_fetch_around_page_gt1_empty_returns_gracefully() -> None: + """fetch_around на page>1 с 0 карточек должен тихо завершить пагинацию (не raise).""" + from app.services.scrapers.avito import AvitoScraper + + scraper = AvitoScraper() + mock_session = AsyncMock() + + # Страница 1: нормальный HTML с одной карточкой + valid_html = ( + "" + '
' + '' + "2-к. квартира, 54,3 м², 5/9 эт." + "" + '' + "
" + "" + ) + # Страница 2: пустой HTML → end of pagination + empty_html = "

no listings

" + + page1_resp = MagicMock() + page1_resp.status_code = 200 + page1_resp.text = valid_html + + page2_resp = MagicMock() + page2_resp.status_code = 200 + page2_resp.text = empty_html + + mock_session.get = AsyncMock(side_effect=[page1_resp, page2_resp]) + scraper._cffi = mock_session + + with patch("app.services.scrapers.avito.AvitoScraper.sleep_between_requests", AsyncMock()): + result = await scraper.fetch_around(56.84, 60.60, 1000, pages=2) + + # Должен вернуть лоты со страницы 1, не raise на странице 2 + assert len(result) >= 1 + + @pytest.mark.asyncio async def test_fetch_around_raises_blocked_on_403() -> None: """AvitoScraper.fetch_around должен raise AvitoBlockedError на HTTP 403.""" @@ -130,7 +206,10 @@ async def test_pipeline_aborts_after_3_consecutive_blocks() -> None: ): with pytest.raises(AvitoBlockedError): await run_avito_pipeline( - mock_db, 56.8, 60.6, 1000, + mock_db, + 56.8, + 60.6, + 1000, enrich_houses=False, enrich_detail_top_n=4, ) @@ -176,7 +255,10 @@ async def test_pipeline_sleep_between_detail_requests() -> None: patch("app.services.scrape_pipeline.random.uniform", return_value=1.0), ): await run_avito_pipeline( - mock_db, 56.8, 60.6, 1000, + mock_db, + 56.8, + 60.6, + 1000, enrich_houses=False, enrich_detail_top_n=3, request_delay_sec=7.0,