bot-backend
f7deed096b
Some checks failed
Deploy Trade-In / changes (push) Successful in 4s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / test (push) Successful in 25s
Deploy Trade-In / build-backend (push) Successful in 1m11s
Deploy Trade-In / deploy (push) Has been cancelled
Co-authored-by: bot-backend <bot-backend@gendsgn.local> Co-committed-by: bot-backend <bot-backend@gendsgn.local>
125 lines
5.3 KiB
Python
125 lines
5.3 KiB
Python
"""#823 — детект HTTP-200 block/captcha страницы и parse_failures счётчик.
|
||
|
||
Тест 1: _is_firewall_page возвращает True на fixtures/avito_blockpage_sample.html.
|
||
Тест 2: _parse_html на блок-странице возвращает 0 лотов (нет карточек).
|
||
Тест 3: fetch_around бросает AvitoContentBlockedError когда page=1 даёт 0 лотов
|
||
(гарантия что silent 0-done НЕ происходит — основной DoD #823).
|
||
Тест 4: _dom_card_to_lot инкрементирует parse_failures при сломанной карточке.
|
||
|
||
Без сети, без БД, без curl_cffi.
|
||
"""
|
||
|
||
import os
|
||
from pathlib import Path
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
from selectolax.parser import HTMLParser
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||
|
||
from app.services.scrapers.avito import AvitoScraper, _is_firewall_page
|
||
from app.services.scrapers.avito_exceptions import AvitoContentBlockedError
|
||
|
||
FIXTURES = Path(__file__).parent / "fixtures"
|
||
BLOCKPAGE_HTML = (FIXTURES / "avito_blockpage_sample.html").read_text(encoding="utf-8")
|
||
SERP_HTML = (FIXTURES / "avito_serp_sample.html").read_text(encoding="utf-8")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 1. _is_firewall_page детектит блок-маркер в HTML
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_is_firewall_page_detects_block_fixture() -> None:
|
||
"""Блок-страница с 'доступ ограничен' и 'firewall-container' → True."""
|
||
assert _is_firewall_page(BLOCKPAGE_HTML) is True
|
||
|
||
|
||
def test_is_firewall_page_false_for_normal_serp() -> None:
|
||
"""Нормальная SERP страница не является блок-страницей."""
|
||
assert _is_firewall_page(SERP_HTML) is False
|
||
|
||
|
||
def test_is_firewall_page_false_for_empty_html() -> None:
|
||
assert _is_firewall_page("") is False
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 2. _parse_html на блок-странице даёт 0 лотов (нет data-marker="item")
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_html_block_page_returns_empty_list() -> None:
|
||
"""Блок-страница не содержит карточек — парсер возвращает []."""
|
||
scraper = AvitoScraper()
|
||
lots = scraper._parse_html(BLOCKPAGE_HTML, "https://www.avito.ru/x")
|
||
assert lots == []
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 3. fetch_around raises AvitoContentBlockedError при 0 лотах на page=1
|
||
# (гарантия что silent 0-done НЕ происходит — основной DoD #823)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_around_raises_on_zero_lots_page1() -> None:
|
||
"""HTTP-200 + пустой SERP (блок/captcha) на page=1 → AvitoContentBlockedError.
|
||
|
||
Мокаем _fetch_serp_html чтобы вернуть блок-страницу без сети.
|
||
"""
|
||
scraper = AvitoScraper()
|
||
with patch.object(
|
||
scraper,
|
||
"_fetch_serp_html",
|
||
new=AsyncMock(return_value=BLOCKPAGE_HTML),
|
||
):
|
||
with pytest.raises(AvitoContentBlockedError):
|
||
await scraper.fetch_around(56.838, 60.605, radius_m=1000, pages=1)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 4. parse_failures счётчик инкрементируется при сломанной карточке
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_parse_failures_starts_at_zero() -> None:
|
||
scraper = AvitoScraper()
|
||
assert scraper.parse_failures == 0
|
||
|
||
|
||
def test_parse_failures_not_incremented_on_link_missing() -> None:
|
||
"""Карточка без item-title link → ранний return None, НЕ через except."""
|
||
scraper = AvitoScraper()
|
||
broken_html = '<div data-marker="item" data-item-id="BROKEN"><span>no link</span></div>'
|
||
tree = HTMLParser(broken_html)
|
||
cards = tree.css('[data-marker="item"]')
|
||
result = scraper._dom_card_to_lot(cards[0], "https://www.avito.ru/x")
|
||
assert result is None
|
||
assert scraper.parse_failures == 0 # ранний return, не exception path
|
||
|
||
|
||
def test_parse_failures_incremented_on_exception() -> None:
|
||
"""css_first бросает TypeError → except path → parse_failures++."""
|
||
scraper = AvitoScraper()
|
||
bad_card = MagicMock()
|
||
bad_card.css_first.side_effect = TypeError("simulated DOM error")
|
||
bad_card.attributes = {}
|
||
|
||
result = scraper._dom_card_to_lot(bad_card, "https://www.avito.ru/x")
|
||
assert result is None
|
||
assert scraper.parse_failures == 1 # #823: счётчик вырос
|
||
|
||
|
||
def test_parse_failures_accumulates_across_cards() -> None:
|
||
"""Несколько сломанных карточек аккумулируются в одном счётчике."""
|
||
scraper = AvitoScraper()
|
||
|
||
for _ in range(3):
|
||
bad_card = MagicMock()
|
||
bad_card.css_first.side_effect = RuntimeError("bad card")
|
||
bad_card.attributes = {}
|
||
scraper._dom_card_to_lot(bad_card, "https://www.avito.ru/x")
|
||
|
||
assert scraper.parse_failures == 3
|