feat(tradein): avito curl_cffi 429 short-retry (backconnect conn-limit transient, no rotate/ban)
This commit is contained in:
parent
8a7538ab13
commit
2c6d0e4c71
2 changed files with 155 additions and 3 deletions
|
|
@ -248,6 +248,13 @@ _AVITO_PRICE_SEED_BRACKETS: list[tuple[int, int | None]] = get_price_seed_bracke
|
|||
# Avito SERP показывает ~50 карточек на страницу.
|
||||
_AVITO_OFFERS_PER_PAGE = 50
|
||||
|
||||
# HTTP 429 в curl_cffi-режиме через backconnect-прокси (mproxy.site) — НЕ IP-ban, а
|
||||
# transient «слишком много одновременных соединений» (лимит 5). Проходит на коротком
|
||||
# retry без ротации IP. Делаем до _AVITO_429_MAX_RETRIES коротких пауз перед тем как
|
||||
# свалиться в старое поведение (rotation / AvitoRateLimitedError).
|
||||
_AVITO_429_MAX_RETRIES = 4
|
||||
_AVITO_429_BACKOFF_SEC = 1.5
|
||||
|
||||
# Счётчик результатов в шапке SERP: <span data-marker="page-title/count">1 178</span>.
|
||||
# Текст может содержать NBSP (\xa0), -entity или обычные пробелы как разделители
|
||||
# разрядов — чистятся в _extract_total_count. [^<]* нежадно ловит весь текст до </span>.
|
||||
|
|
@ -394,6 +401,7 @@ class AvitoScraper(BaseScraper):
|
|||
max_rot = settings.avito_proxy_max_rotations if settings.avito_proxy_rotate_url else 0
|
||||
rot_done = 0
|
||||
transient_left = 2
|
||||
r429 = 0
|
||||
while True:
|
||||
try:
|
||||
response = await self._cffi.get(url)
|
||||
|
|
@ -412,6 +420,20 @@ class AvitoScraper(BaseScraper):
|
|||
|
||||
sc = response.status_code
|
||||
is_firewall = sc == 200 and _is_firewall_page(response.text)
|
||||
# 429 в curl_cffi через backconnect-прокси — transient conn-limit, НЕ
|
||||
# IP-ban: короткий retry того же url без ротации. Только если короткие
|
||||
# retry исчерпаны — проваливаемся в общую блок-логику ниже (rotation/raise).
|
||||
if sc == 429 and r429 < _AVITO_429_MAX_RETRIES:
|
||||
r429 += 1
|
||||
logger.info(
|
||||
"avito page=%d HTTP 429 (backconnect conn-limit) — short retry %d/%d",
|
||||
page,
|
||||
r429,
|
||||
_AVITO_429_MAX_RETRIES,
|
||||
)
|
||||
jitter = (r429 % 3) * 0.1
|
||||
await asyncio.sleep(_AVITO_429_BACKOFF_SEC + jitter)
|
||||
continue
|
||||
if sc in (403, 429) or is_firewall:
|
||||
if rot_done < max_rot and await self._rotate_ip():
|
||||
rot_done += 1
|
||||
|
|
|
|||
|
|
@ -151,7 +151,11 @@ async def test_fetch_around_raises_blocked_on_403() -> None:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_around_raises_rate_limited_on_429() -> None:
|
||||
"""AvitoScraper.fetch_around должен raise AvitoRateLimitedError на HTTP 429."""
|
||||
"""AvitoScraper.fetch_around должен raise AvitoRateLimitedError на persistent HTTP 429.
|
||||
|
||||
429 (backconnect conn-limit) сначала проходит короткие retry; при отсутствии
|
||||
rotate-url после исчерпания retry сохраняется старый fallback — AvitoRateLimitedError.
|
||||
"""
|
||||
from app.services.scrapers.avito import AvitoScraper
|
||||
|
||||
scraper = AvitoScraper()
|
||||
|
|
@ -161,8 +165,9 @@ async def test_fetch_around_raises_rate_limited_on_429() -> None:
|
|||
mock_session.get = AsyncMock(return_value=mock_response)
|
||||
scraper._cffi = mock_session
|
||||
|
||||
with pytest.raises(AvitoRateLimitedError):
|
||||
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
|
||||
with patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()):
|
||||
with pytest.raises(AvitoRateLimitedError):
|
||||
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
|
||||
|
||||
|
||||
# ── Pipeline abort tests ────────────────────────────────────────
|
||||
|
|
@ -431,3 +436,128 @@ def test_avito_proxies_helper(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
url = "http://u:p@mproxy.site:16928"
|
||||
monkeypatch.setattr(sp.settings, "avito_proxy_url", url)
|
||||
assert sp._avito_proxies() == {"http": url, "https": url}
|
||||
|
||||
|
||||
# ── curl_cffi 429 short-retry (backconnect conn-limit transient) ──────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_around_429_short_retry_then_succeeds() -> None:
|
||||
"""429 дважды → 200: короткий retry снимает transient conn-limit БЕЗ ротации IP.
|
||||
|
||||
backconnect-прокси (mproxy.site, лимит 5 соединений) отдаёт 429 при параллельной
|
||||
пагинации — это transient, проходит на коротком retry. _rotate_ip НЕ должен дёргаться.
|
||||
"""
|
||||
from app.services.scrapers.avito import AvitoScraper
|
||||
|
||||
scraper = AvitoScraper()
|
||||
r429 = MagicMock()
|
||||
r429.status_code = 429
|
||||
ok = MagicMock()
|
||||
ok.status_code = 200
|
||||
ok.text = _CARD_HTML
|
||||
mock_session = AsyncMock()
|
||||
mock_session.get = AsyncMock(side_effect=[r429, r429, ok])
|
||||
scraper._cffi = mock_session
|
||||
|
||||
with (
|
||||
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()) as slp,
|
||||
patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot,
|
||||
patch.object(AvitoScraper, "sleep_between_requests", AsyncMock()),
|
||||
):
|
||||
result = await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
|
||||
|
||||
assert len(result) == 1
|
||||
assert mock_session.get.await_count == 3 # 429, 429, 200
|
||||
rot.assert_not_awaited() # короткий retry, не ротация
|
||||
# короткий backoff (_AVITO_429_BACKOFF_SEC ~1.5s), не 9с-settle ротации
|
||||
assert slp.await_count >= 2
|
||||
for c in slp.await_args_list:
|
||||
assert c.args[0] < 5.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_around_429_exhausted_no_rotate_url_raises() -> None:
|
||||
"""429 постоянно + НЕТ rotate-url → после исчерпания коротких retry: старый fallback."""
|
||||
from app.services.scrapers import avito as avito_mod
|
||||
from app.services.scrapers.avito import AvitoScraper
|
||||
|
||||
scraper = AvitoScraper()
|
||||
r429 = MagicMock()
|
||||
r429.status_code = 429
|
||||
mock_session = AsyncMock()
|
||||
mock_session.get = AsyncMock(return_value=r429)
|
||||
scraper._cffi = mock_session
|
||||
|
||||
with (
|
||||
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()) as slp,
|
||||
patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot,
|
||||
):
|
||||
with pytest.raises(AvitoRateLimitedError):
|
||||
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
|
||||
|
||||
rot.assert_not_awaited() # нет rotate-url → ротация не дёргается
|
||||
# ровно _AVITO_429_MAX_RETRIES коротких retry, потом fallback
|
||||
assert slp.await_count == avito_mod._AVITO_429_MAX_RETRIES
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_around_429_exhausted_with_rotate_url_rotates(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""429 постоянно + ЕСТЬ rotate-url → короткие retry, затем ротация (старый путь)."""
|
||||
from app.services.scrapers import avito as avito_mod
|
||||
from app.services.scrapers.avito import AvitoScraper
|
||||
|
||||
monkeypatch.setattr(
|
||||
avito_mod.settings, "avito_proxy_rotate_url", "https://changeip.example/?proxy_key=x"
|
||||
)
|
||||
monkeypatch.setattr(avito_mod.settings, "avito_proxy_max_rotations", 1)
|
||||
|
||||
scraper = AvitoScraper()
|
||||
r429 = MagicMock()
|
||||
r429.status_code = 429
|
||||
ok = MagicMock()
|
||||
ok.status_code = 200
|
||||
ok.text = _CARD_HTML
|
||||
mock_session = AsyncMock()
|
||||
# 429 на коротких retry, после ротации — 200
|
||||
mock_session.get = AsyncMock(side_effect=[r429, r429, r429, r429, r429, ok])
|
||||
scraper._cffi = mock_session
|
||||
|
||||
with (
|
||||
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()),
|
||||
patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot,
|
||||
patch.object(AvitoScraper, "sleep_between_requests", AsyncMock()),
|
||||
):
|
||||
result = await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
|
||||
|
||||
rot.assert_awaited_once() # после исчерпания коротких retry — ротация
|
||||
assert len(result) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_around_403_no_429_retry_path(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""403 (реальный IP-ban) НЕ проходит через 429-retry — поведение без изменений."""
|
||||
from app.services.scrapers import avito as avito_mod
|
||||
from app.services.scrapers.avito import AvitoScraper
|
||||
|
||||
monkeypatch.setattr(avito_mod.settings, "avito_proxy_rotate_url", None)
|
||||
|
||||
scraper = AvitoScraper()
|
||||
r403 = MagicMock()
|
||||
r403.status_code = 403
|
||||
mock_session = AsyncMock()
|
||||
mock_session.get = AsyncMock(return_value=r403)
|
||||
scraper._cffi = mock_session
|
||||
|
||||
with (
|
||||
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()) as slp,
|
||||
patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)),
|
||||
):
|
||||
with pytest.raises(AvitoBlockedError):
|
||||
await scraper.fetch_around(56.84, 60.60, 1000, pages=1)
|
||||
|
||||
# 403 без rotate-url → немедленный raise, ни одного 429-backoff sleep
|
||||
slp.assert_not_awaited()
|
||||
assert mock_session.get.await_count == 1
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue