lekss361/gendesign
1
0
Fork 0
Code Issues 57 Pull requests Projects Releases Packages Wiki Activity Actions

fix(tradein): avito content-block (HTTP-200) не маскируется под end-of-pagination (Refs #754) #779

Merged
bot-reviewer merged 1 commit from fix/754-avito-content-block into main 2026-05-30 16:28:23 +00:00
Conversation 0 Commits 1 Files changed 3 +106 -3
3 changed files with 106 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
tradein-mvp/backend/app/services/scrapers/avito.py
View file

@ -25,7 +25,11 @@ from curl_cffi.requests import AsyncSession
from selectolax.parser import HTMLParser
from app.services.scraper_settings import get_scraper_delay
from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
from app.services.scrapers.avito_exceptions import (
AvitoBlockedError,
AvitoContentBlockedError,
AvitoRateLimitedError,
)
from app.services.scrapers.base import BaseScraper, ScrapedLot
logger = logging.getLogger(__name__)
@ -261,6 +265,15 @@ class AvitoScraper(BaseScraper):
lots = self._parse_html(response.text, source_url_base=url)
if not lots:
if page == 1:
logger.error(
"avito SERP page=1 returned HTTP 200 but 0 cards — "
"likely content-block/captcha url=%s",
url,
)
raise AvitoContentBlockedError(
"Avito SERP HTTP 200 with 0 cards on page=1 — content-block suspected"
)
logger.info("avito page=%d: 0 lots — end of pagination", page)
break
all_lots.extend(lots)

8
tradein-mvp/backend/app/services/scrapers/avito_exceptions.py
View file

@ -13,5 +13,13 @@ class AvitoRateLimitedError(AvitoError):
"""HTTP 429 от Avito — rate limit triggered."""
class AvitoContentBlockedError(AvitoBlockedError):
"""HTTP 200 от Avito, но 0 карточек на первой странице — content-block / captcha.
Наследует AvitoBlockedError, чтобы существующие except-блоки на AvitoBlockedError
продолжали ловить этот сигнал (mark_banned, pipeline abort и т.д.).
"""
class AvitoParseError(AvitoError):
"""Cannot parse Avito HTML structure (selector changes)."""

86
tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py
View file

@ -6,6 +6,7 @@ import pytest
from app.services.scrapers.avito_exceptions import (
AvitoBlockedError,
AvitoContentBlockedError,
AvitoRateLimitedError,
)
@ -57,6 +58,81 @@ async def test_fetch_detail_raises_value_error_on_other_non_200() -> None:
# ── SERP (avito.py) tests ───────────────────────────────────────
@pytest.mark.asyncio
async def test_fetch_around_raises_content_blocked_on_page1_empty_html() -> None:
"""fetch_around должен raise AvitoContentBlockedError на HTTP 200 + 0 карточек на странице 1."""
from app.services.scrapers.avito import AvitoScraper
scraper = AvitoScraper()
mock_session = AsyncMock()
mock_response = MagicMock()
mock_response.status_code = 200
# Garbage HTML: нет data-marker="item" → _parse_html вернёт []
mock_response.text = "<html><body><p>captcha challenge</p></body></html>"
mock_session.get = AsyncMock(return_value=mock_response)
scraper._cffi = mock_session
with pytest.raises(AvitoContentBlockedError, match="content-block"):
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
@pytest.mark.asyncio
async def test_fetch_around_content_blocked_is_subtype_of_blocked() -> None:
"""AvitoContentBlockedError должен быть поймать как AvitoBlockedError (наследование)."""
from app.services.scrapers.avito import AvitoScraper
scraper = AvitoScraper()
mock_session = AsyncMock()
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.text = "<html><body></body></html>"
mock_session.get = AsyncMock(return_value=mock_response)
scraper._cffi = mock_session
with pytest.raises(AvitoBlockedError):
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
@pytest.mark.asyncio
async def test_fetch_around_page_gt1_empty_returns_gracefully() -> None:
"""fetch_around на page>1 с 0 карточек должен тихо завершить пагинацию (не raise)."""
from app.services.scrapers.avito import AvitoScraper
scraper = AvitoScraper()
mock_session = AsyncMock()
# Страница 1: нормальный HTML с одной карточкой
valid_html = (
"<html><body>"
'<div data-marker="item" data-item-id="123456789">'
'<a data-marker="item-title" href="/ekaterinburg/kvartiry/test-123456789">'
"2-к. квартира, 54,3 м², 5/9 эт."
"</a>"
'<meta itemprop="price" content="4500000"/>'
"</div>"
"</body></html>"
)
# Страница 2: пустой HTML → end of pagination
empty_html = "<html><body><p>no listings</p></body></html>"
page1_resp = MagicMock()
page1_resp.status_code = 200
page1_resp.text = valid_html
page2_resp = MagicMock()
page2_resp.status_code = 200
page2_resp.text = empty_html
mock_session.get = AsyncMock(side_effect=[page1_resp, page2_resp])
scraper._cffi = mock_session
with patch("app.services.scrapers.avito.AvitoScraper.sleep_between_requests", AsyncMock()):
result = await scraper.fetch_around(56.84, 60.60, 1000, pages=2)
# Должен вернуть лоты со страницы 1, не raise на странице 2
assert len(result) >= 1
@pytest.mark.asyncio
async def test_fetch_around_raises_blocked_on_403() -> None:
"""AvitoScraper.fetch_around должен raise AvitoBlockedError на HTTP 403."""
@ -130,7 +206,10 @@ async def test_pipeline_aborts_after_3_consecutive_blocks() -> None:
):
with pytest.raises(AvitoBlockedError):
await run_avito_pipeline(
mock_db, 56.8, 60.6, 1000,
mock_db,
56.8,
60.6,
1000,
enrich_houses=False,
enrich_detail_top_n=4,
)
@ -176,7 +255,10 @@ async def test_pipeline_sleep_between_detail_requests() -> None:
patch("app.services.scrape_pipeline.random.uniform", return_value=1.0),
):
await run_avito_pipeline(
mock_db, 56.8, 60.6, 1000,
mock_db,
56.8,
60.6,
1000,
enrich_houses=False,
enrich_detail_top_n=3,
request_delay_sec=7.0,