fix(tradein): avito content-block (HTTP-200) не маскируется под end-of-pagination (Refs #754)
_parse_html=[] при HTTP 200 на page==1 (анкер EKB всегда имеет объявления) ⇒ content-block (captcha/challenge) → logger.error + raise AvitoContentBlockedError (subtype AvitoBlockedError — existing except ловит). page>1 с 0 карточек → легитимный end-of-pagination (graceful break, сохранено). 403/429-обработка не тронута. pytest -k avito 101 passed, ruff clean.
This commit is contained in:
parent
79a79004cf
commit
5b430d3ffe
3 changed files with 106 additions and 3 deletions
|
|
@ -25,7 +25,11 @@ from curl_cffi.requests import AsyncSession
|
||||||
from selectolax.parser import HTMLParser
|
from selectolax.parser import HTMLParser
|
||||||
|
|
||||||
from app.services.scraper_settings import get_scraper_delay
|
from app.services.scraper_settings import get_scraper_delay
|
||||||
from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
|
from app.services.scrapers.avito_exceptions import (
|
||||||
|
AvitoBlockedError,
|
||||||
|
AvitoContentBlockedError,
|
||||||
|
AvitoRateLimitedError,
|
||||||
|
)
|
||||||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
@ -261,6 +265,15 @@ class AvitoScraper(BaseScraper):
|
||||||
|
|
||||||
lots = self._parse_html(response.text, source_url_base=url)
|
lots = self._parse_html(response.text, source_url_base=url)
|
||||||
if not lots:
|
if not lots:
|
||||||
|
if page == 1:
|
||||||
|
logger.error(
|
||||||
|
"avito SERP page=1 returned HTTP 200 but 0 cards — "
|
||||||
|
"likely content-block/captcha url=%s",
|
||||||
|
url,
|
||||||
|
)
|
||||||
|
raise AvitoContentBlockedError(
|
||||||
|
"Avito SERP HTTP 200 with 0 cards on page=1 — content-block suspected"
|
||||||
|
)
|
||||||
logger.info("avito page=%d: 0 lots — end of pagination", page)
|
logger.info("avito page=%d: 0 lots — end of pagination", page)
|
||||||
break
|
break
|
||||||
all_lots.extend(lots)
|
all_lots.extend(lots)
|
||||||
|
|
|
||||||
|
|
@ -13,5 +13,13 @@ class AvitoRateLimitedError(AvitoError):
|
||||||
"""HTTP 429 от Avito — rate limit triggered."""
|
"""HTTP 429 от Avito — rate limit triggered."""
|
||||||
|
|
||||||
|
|
||||||
|
class AvitoContentBlockedError(AvitoBlockedError):
|
||||||
|
"""HTTP 200 от Avito, но 0 карточек на первой странице — content-block / captcha.
|
||||||
|
|
||||||
|
Наследует AvitoBlockedError, чтобы существующие except-блоки на AvitoBlockedError
|
||||||
|
продолжали ловить этот сигнал (mark_banned, pipeline abort и т.д.).
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
class AvitoParseError(AvitoError):
|
class AvitoParseError(AvitoError):
|
||||||
"""Cannot parse Avito HTML structure (selector changes)."""
|
"""Cannot parse Avito HTML structure (selector changes)."""
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ import pytest
|
||||||
|
|
||||||
from app.services.scrapers.avito_exceptions import (
|
from app.services.scrapers.avito_exceptions import (
|
||||||
AvitoBlockedError,
|
AvitoBlockedError,
|
||||||
|
AvitoContentBlockedError,
|
||||||
AvitoRateLimitedError,
|
AvitoRateLimitedError,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -57,6 +58,81 @@ async def test_fetch_detail_raises_value_error_on_other_non_200() -> None:
|
||||||
# ── SERP (avito.py) tests ───────────────────────────────────────
|
# ── SERP (avito.py) tests ───────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fetch_around_raises_content_blocked_on_page1_empty_html() -> None:
|
||||||
|
"""fetch_around должен raise AvitoContentBlockedError на HTTP 200 + 0 карточек на странице 1."""
|
||||||
|
from app.services.scrapers.avito import AvitoScraper
|
||||||
|
|
||||||
|
scraper = AvitoScraper()
|
||||||
|
mock_session = AsyncMock()
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.status_code = 200
|
||||||
|
# Garbage HTML: нет data-marker="item" → _parse_html вернёт []
|
||||||
|
mock_response.text = "<html><body><p>captcha challenge</p></body></html>"
|
||||||
|
mock_session.get = AsyncMock(return_value=mock_response)
|
||||||
|
scraper._cffi = mock_session
|
||||||
|
|
||||||
|
with pytest.raises(AvitoContentBlockedError, match="content-block"):
|
||||||
|
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fetch_around_content_blocked_is_subtype_of_blocked() -> None:
|
||||||
|
"""AvitoContentBlockedError должен быть поймать как AvitoBlockedError (наследование)."""
|
||||||
|
from app.services.scrapers.avito import AvitoScraper
|
||||||
|
|
||||||
|
scraper = AvitoScraper()
|
||||||
|
mock_session = AsyncMock()
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.status_code = 200
|
||||||
|
mock_response.text = "<html><body></body></html>"
|
||||||
|
mock_session.get = AsyncMock(return_value=mock_response)
|
||||||
|
scraper._cffi = mock_session
|
||||||
|
|
||||||
|
with pytest.raises(AvitoBlockedError):
|
||||||
|
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fetch_around_page_gt1_empty_returns_gracefully() -> None:
|
||||||
|
"""fetch_around на page>1 с 0 карточек должен тихо завершить пагинацию (не raise)."""
|
||||||
|
from app.services.scrapers.avito import AvitoScraper
|
||||||
|
|
||||||
|
scraper = AvitoScraper()
|
||||||
|
mock_session = AsyncMock()
|
||||||
|
|
||||||
|
# Страница 1: нормальный HTML с одной карточкой
|
||||||
|
valid_html = (
|
||||||
|
"<html><body>"
|
||||||
|
'<div data-marker="item" data-item-id="123456789">'
|
||||||
|
'<a data-marker="item-title" href="/ekaterinburg/kvartiry/test-123456789">'
|
||||||
|
"2-к. квартира, 54,3 м², 5/9 эт."
|
||||||
|
"</a>"
|
||||||
|
'<meta itemprop="price" content="4500000"/>'
|
||||||
|
"</div>"
|
||||||
|
"</body></html>"
|
||||||
|
)
|
||||||
|
# Страница 2: пустой HTML → end of pagination
|
||||||
|
empty_html = "<html><body><p>no listings</p></body></html>"
|
||||||
|
|
||||||
|
page1_resp = MagicMock()
|
||||||
|
page1_resp.status_code = 200
|
||||||
|
page1_resp.text = valid_html
|
||||||
|
|
||||||
|
page2_resp = MagicMock()
|
||||||
|
page2_resp.status_code = 200
|
||||||
|
page2_resp.text = empty_html
|
||||||
|
|
||||||
|
mock_session.get = AsyncMock(side_effect=[page1_resp, page2_resp])
|
||||||
|
scraper._cffi = mock_session
|
||||||
|
|
||||||
|
with patch("app.services.scrapers.avito.AvitoScraper.sleep_between_requests", AsyncMock()):
|
||||||
|
result = await scraper.fetch_around(56.84, 60.60, 1000, pages=2)
|
||||||
|
|
||||||
|
# Должен вернуть лоты со страницы 1, не raise на странице 2
|
||||||
|
assert len(result) >= 1
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_fetch_around_raises_blocked_on_403() -> None:
|
async def test_fetch_around_raises_blocked_on_403() -> None:
|
||||||
"""AvitoScraper.fetch_around должен raise AvitoBlockedError на HTTP 403."""
|
"""AvitoScraper.fetch_around должен raise AvitoBlockedError на HTTP 403."""
|
||||||
|
|
@ -130,7 +206,10 @@ async def test_pipeline_aborts_after_3_consecutive_blocks() -> None:
|
||||||
):
|
):
|
||||||
with pytest.raises(AvitoBlockedError):
|
with pytest.raises(AvitoBlockedError):
|
||||||
await run_avito_pipeline(
|
await run_avito_pipeline(
|
||||||
mock_db, 56.8, 60.6, 1000,
|
mock_db,
|
||||||
|
56.8,
|
||||||
|
60.6,
|
||||||
|
1000,
|
||||||
enrich_houses=False,
|
enrich_houses=False,
|
||||||
enrich_detail_top_n=4,
|
enrich_detail_top_n=4,
|
||||||
)
|
)
|
||||||
|
|
@ -176,7 +255,10 @@ async def test_pipeline_sleep_between_detail_requests() -> None:
|
||||||
patch("app.services.scrape_pipeline.random.uniform", return_value=1.0),
|
patch("app.services.scrape_pipeline.random.uniform", return_value=1.0),
|
||||||
):
|
):
|
||||||
await run_avito_pipeline(
|
await run_avito_pipeline(
|
||||||
mock_db, 56.8, 60.6, 1000,
|
mock_db,
|
||||||
|
56.8,
|
||||||
|
60.6,
|
||||||
|
1000,
|
||||||
enrich_houses=False,
|
enrich_houses=False,
|
||||||
enrich_detail_top_n=3,
|
enrich_detail_top_n=3,
|
||||||
request_delay_sec=7.0,
|
request_delay_sec=7.0,
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue