bot-backend
bee44ffd3d
Run 216 recovered 403s (merged fix) but still banned on a 429 at page 36 (deep pagination, backconnect conn-limit). Widen the 429 retry budget + one reconnect on exhaustion in _fetch_serp_html, and make fetch_all_secondary skip a blocked bucket and continue rather than aborting the run; only N consecutive blocked buckets (real hard ban) re-raises. Preserves hard-ban detection.
242 lines
9.9 KiB
Python
242 lines
9.9 KiB
Python
"""429-hardening: widened short-retry + reconnect-fallback в _fetch_serp_html и
|
||
per-bucket skip в fetch_all_secondary.
|
||
|
||
Run 216 (прод): merged 403-reconnect-retry восстановил два 403, но прогон всё равно
|
||
забанился на HTTP 429 на page 36 (deep-pagination dense-бакета). 429 под backconnect =
|
||
transient conn-limit (лимит 5 соединений), не IP-ban.
|
||
|
||
Две защиты:
|
||
A. _fetch_serp_html: short-retry того же session расширен до _AVITO_429_MAX_RETRIES=8;
|
||
на исчерпании под backconnect — ещё _AVITO_429_RECONNECT_RETRIES попыток через
|
||
_reset_cffi (освобождает conn-slot) перед AvitoRateLimitedError.
|
||
B. fetch_all_secondary: одиночный заблокированный бакет скипается (continue), sweep
|
||
продолжается; только _AVITO_SWEEP_MAX_CONSECUTIVE_BLOCKED ПОДРЯД блоков = hard ban
|
||
→ re-raise (сохраняет real-ban detection).
|
||
"""
|
||
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers import avito as avito_mod
|
||
from app.services.scrapers.avito import AvitoScraper
|
||
from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
|
||
|
||
# Валидная карточка SERP (одна) — successful page.
|
||
_CARD_HTML = (
|
||
"<html><body>"
|
||
'<div data-marker="item" data-item-id="123456789">'
|
||
'<a data-marker="item-title" href="/ekaterinburg/kvartiry/test-123456789">'
|
||
"2-к. квартира, 54,3 м², 5/9 эт."
|
||
"</a>"
|
||
'<meta itemprop="price" content="4500000"/>'
|
||
"</div>"
|
||
"</body></html>"
|
||
)
|
||
|
||
|
||
def _enable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""scraper_proxy_url задан (через env-поле), changeip-URL — нет → backconnect."""
|
||
monkeypatch.setattr(avito_mod.settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619")
|
||
monkeypatch.setattr(avito_mod.settings, "avito_proxy_url", None)
|
||
monkeypatch.setattr(avito_mod.settings, "avito_proxy_rotate_url", None)
|
||
|
||
|
||
def _resp(status: int, text: str = "") -> MagicMock:
|
||
r = MagicMock()
|
||
r.status_code = status
|
||
r.text = text
|
||
return r
|
||
|
||
|
||
# --- A. _fetch_serp_html 429 short-retry + reconnect-fallback -------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_429_short_retries_then_200_within_widened_budget(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""429×6 (< _AVITO_429_MAX_RETRIES=8) → 200: short-retry того же session снимает,
|
||
_reset_cffi НЕ дёргается (reconnect-fallback не нужен)."""
|
||
_enable_backconnect(monkeypatch)
|
||
n_429 = 6
|
||
assert n_429 < avito_mod._AVITO_429_MAX_RETRIES
|
||
|
||
scraper = AvitoScraper()
|
||
seq = [_resp(429) for _ in range(n_429)] + [_resp(200, _CARD_HTML)]
|
||
mock_session = AsyncMock()
|
||
mock_session.get = AsyncMock(side_effect=seq)
|
||
scraper._cffi = mock_session
|
||
|
||
reset = AsyncMock()
|
||
with (
|
||
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()),
|
||
patch.object(AvitoScraper, "_reset_cffi", reset),
|
||
):
|
||
html = await scraper._fetch_serp_html("https://www.avito.ru/x", page=36)
|
||
|
||
assert html == _CARD_HTML
|
||
reset.assert_not_awaited() # short-retry хватило, reconnect не понадобился
|
||
assert mock_session.get.await_count == n_429 + 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_429_exhausted_then_reconnect_then_200(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""short-retry исчерпан (429×_AVITO_429_MAX_RETRIES) → reconnect-fallback (_reset_cffi)
|
||
→ 200 успех. Подтверждает: одна reconnect-фаза освобождает conn-slot."""
|
||
_enable_backconnect(monkeypatch)
|
||
n_short = avito_mod._AVITO_429_MAX_RETRIES
|
||
|
||
scraper = AvitoScraper()
|
||
# _AVITO_429_MAX_RETRIES коротких retry (все 429), ещё один 429 проваливается в
|
||
# reconnect-block, после _reset_cffi следующий ответ 200.
|
||
seq = [_resp(429) for _ in range(n_short + 1)] + [_resp(200, _CARD_HTML)]
|
||
mock_session = AsyncMock()
|
||
mock_session.get = AsyncMock(side_effect=seq)
|
||
scraper._cffi = mock_session
|
||
|
||
reset = AsyncMock()
|
||
with (
|
||
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()),
|
||
patch.object(AvitoScraper, "_reset_cffi", reset),
|
||
):
|
||
html = await scraper._fetch_serp_html("https://www.avito.ru/x", page=36)
|
||
|
||
assert html == _CARD_HTML
|
||
reset.assert_awaited() # reconnect-fallback сработал
|
||
assert reset.await_count == 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_429_fully_exhausted_raises_rate_limited(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""429 на каждый запрос: short-retry + все reconnect-fallback исчерпаны →
|
||
AvitoRateLimitedError. _reset_cffi дёрнут ровно _AVITO_429_RECONNECT_RETRIES раз."""
|
||
_enable_backconnect(monkeypatch)
|
||
|
||
scraper = AvitoScraper()
|
||
mock_session = AsyncMock()
|
||
mock_session.get = AsyncMock(return_value=_resp(429))
|
||
scraper._cffi = mock_session
|
||
|
||
reset = AsyncMock()
|
||
with (
|
||
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()),
|
||
patch.object(AvitoScraper, "_reset_cffi", reset),
|
||
):
|
||
with pytest.raises(AvitoRateLimitedError, match="rate limited"):
|
||
await scraper._fetch_serp_html("https://www.avito.ru/x", page=36)
|
||
|
||
assert reset.await_count == avito_mod._AVITO_429_RECONNECT_RETRIES
|
||
# get: 1 первичный + _AVITO_429_MAX_RETRIES short-retry + _AVITO_429_RECONNECT_RETRIES
|
||
assert (
|
||
mock_session.get.await_count
|
||
== 1 + avito_mod._AVITO_429_MAX_RETRIES + avito_mod._AVITO_429_RECONNECT_RETRIES
|
||
)
|
||
|
||
|
||
# --- B. fetch_all_secondary per-bucket skip -----------------------------------
|
||
|
||
|
||
def _single_room() -> list[tuple[str, str]]:
|
||
"""Один room-bucket → число брекетов = len(_AVITO_PRICE_SEED_BRACKETS)."""
|
||
return [("2-комнатные", "2-komnatnye")]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_one_blocked_bucket_skipped_sweep_completes(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""_walk_price_range кидает AvitoRateLimitedError на 1-м брекете, успех на остальных →
|
||
sweep завершается без raise, заблокированный бакет посчитан."""
|
||
n_brackets = len(avito_mod._AVITO_PRICE_SEED_BRACKETS)
|
||
assert n_brackets >= 2
|
||
|
||
scraper = AvitoScraper()
|
||
calls = {"n": 0}
|
||
|
||
async def fake_walk(self: object, **kwargs: object) -> None:
|
||
calls["n"] += 1
|
||
if calls["n"] == 1:
|
||
raise AvitoRateLimitedError("429 deep pagination")
|
||
# успех: ничего не добавляем в seen (не нужно для теста контракта)
|
||
|
||
with patch.object(AvitoScraper, "_walk_price_range", fake_walk):
|
||
result = await scraper.fetch_all_secondary(rooms_buckets=_single_room())
|
||
|
||
assert result == [] # seen пуст, но без raise
|
||
assert calls["n"] == n_brackets # все брекеты пройдены (1 скипнут, остальные ок)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_consecutive_blocked_threshold_reraises(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""_walk_price_range кидает _AVITO_SWEEP_MAX_CONSECUTIVE_BLOCKED раз ПОДРЯД →
|
||
последнее исключение пробрасывается (real hard-ban detection сохранена)."""
|
||
thr = avito_mod._AVITO_SWEEP_MAX_CONSECUTIVE_BLOCKED
|
||
assert len(avito_mod._AVITO_PRICE_SEED_BRACKETS) >= thr
|
||
|
||
scraper = AvitoScraper()
|
||
|
||
async def always_block(self: object, **kwargs: object) -> None:
|
||
raise AvitoBlockedError("IP banned")
|
||
|
||
with patch.object(AvitoScraper, "_walk_price_range", always_block):
|
||
with pytest.raises(AvitoBlockedError, match="IP banned"):
|
||
await scraper.fetch_all_secondary(rooms_buckets=_single_room())
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_success_between_blocks_resets_counter(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Чередование block/success НЕ срабатывает порог: успех между блоками сбрасывает
|
||
consecutive_blocked. Паттерн block,success,block,success,... → sweep завершается."""
|
||
thr = avito_mod._AVITO_SWEEP_MAX_CONSECUTIVE_BLOCKED
|
||
n_brackets = len(avito_mod._AVITO_PRICE_SEED_BRACKETS)
|
||
# Нужно достаточно брекетов чтобы суммарно было > thr блоков, но не подряд.
|
||
assert n_brackets > thr
|
||
|
||
scraper = AvitoScraper()
|
||
calls = {"n": 0}
|
||
|
||
async def alternate(self: object, **kwargs: object) -> None:
|
||
calls["n"] += 1
|
||
# нечётные вызовы блокируются, чётные — успех → никогда не thr подряд
|
||
if calls["n"] % 2 == 1:
|
||
raise AvitoRateLimitedError("429 transient")
|
||
|
||
with patch.object(AvitoScraper, "_walk_price_range", alternate):
|
||
result = await scraper.fetch_all_secondary(rooms_buckets=_single_room())
|
||
|
||
assert result == [] # без raise — порог не пробит
|
||
assert calls["n"] == n_brackets
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_incremental_path_also_skips_blocked_bucket(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Per-bucket skip применяется и к incremental-пути (_paginate_incremental_bracket)."""
|
||
from datetime import date
|
||
|
||
n_brackets = len(avito_mod._AVITO_PRICE_SEED_BRACKETS)
|
||
scraper = AvitoScraper()
|
||
calls = {"n": 0}
|
||
|
||
async def fake_incr(self: object, **kwargs: object) -> None:
|
||
calls["n"] += 1
|
||
if calls["n"] == 1:
|
||
raise AvitoRateLimitedError("429")
|
||
|
||
with patch.object(AvitoScraper, "_paginate_incremental_bracket", fake_incr):
|
||
result = await scraper.fetch_all_secondary(
|
||
rooms_buckets=_single_room(), since=date(2026, 1, 1)
|
||
)
|
||
|
||
assert result == []
|
||
assert calls["n"] == n_brackets
|