bot-backend
44e316c514
PR #1764's reconnect-retry was dead code in prod: the gate excluded configs with avito_proxy_rotate_url set, but prod has BOTH the backconnect SCRAPER_PROXY_URL (what curl_cffi egresses through) AND a leftover changeip URL for the separate auv/browser proxy. With max_rotations=0 neither path fired -> a single SERP 403 still aborted the sweep (run 212). The curl session always uses scraper_proxy_url, so session-rebuild rotates the exit IP regardless of changeip config.
220 lines
9 KiB
Python
220 lines
9 KiB
Python
"""curl_cffi 403/firewall reconnect-retry под backconnect-прокси.
|
||
|
||
Под backconnect rotating-прокси (mproxy.site, нет changeip-URL) одиночный HTTP 403
|
||
или firewall-интерстициал ронял весь avito_full_load sweep через AvitoBlockedError →
|
||
mark_banned. Каждое новое соединение через backconnect даёт свежий exit-IP, поэтому
|
||
ретраим пересозданием curl_cffi-сессии до _AVITO_403_MAX_RETRIES раз.
|
||
|
||
curl_cffi-сессия всегда egress через scraper_proxy_url (backconnect mproxy), поэтому
|
||
reconnect-retry гейтим ТОЛЬКО на наличие proxy — независимо от того, задан ли changeip-URL
|
||
(тот относится к ДРУГОМУ, browser/auv-прокси). Включаем через scraper_proxy_url_env
|
||
(scraper_proxy_url — read-only property, его не monkeypatch'ишь).
|
||
"""
|
||
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from app.services.scrapers import avito as avito_mod
|
||
from app.services.scrapers.avito import AvitoScraper
|
||
from app.services.scrapers.avito_exceptions import AvitoBlockedError
|
||
|
||
# Валидная карточка SERP (одна) — successful page после reconnect.
|
||
_CARD_HTML = (
|
||
"<html><body>"
|
||
'<div data-marker="item" data-item-id="123456789">'
|
||
'<a data-marker="item-title" href="/ekaterinburg/kvartiry/test-123456789">'
|
||
"2-к. квартира, 54,3 м², 5/9 эт."
|
||
"</a>"
|
||
'<meta itemprop="price" content="4500000"/>'
|
||
"</div>"
|
||
"</body></html>"
|
||
)
|
||
|
||
# Firewall-интерстициал (HTTP 200, не 403).
|
||
_FIREWALL_HTML = (
|
||
"<html><head><title>Доступ ограничен: проблема с IP</title></head>"
|
||
'<body><div class="firewall-container">...</div></body></html>'
|
||
)
|
||
|
||
|
||
def _enable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""scraper_proxy_url задан (через env-поле), changeip-URL — нет → backconnect."""
|
||
monkeypatch.setattr(avito_mod.settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619")
|
||
monkeypatch.setattr(avito_mod.settings, "avito_proxy_url", None)
|
||
monkeypatch.setattr(avito_mod.settings, "avito_proxy_rotate_url", None)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_backconnect_403_403_200_reconnect_succeeds(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""backconnect: 403 → 403 → 200 → reconnect-retry возвращает HTML, _reset_cffi дёрнут 2×."""
|
||
_enable_backconnect(monkeypatch)
|
||
|
||
scraper = AvitoScraper()
|
||
r403 = MagicMock()
|
||
r403.status_code = 403
|
||
r403.text = ""
|
||
ok = MagicMock()
|
||
ok.status_code = 200
|
||
ok.text = _CARD_HTML
|
||
mock_session = AsyncMock()
|
||
mock_session.get = AsyncMock(side_effect=[r403, r403, ok])
|
||
scraper._cffi = mock_session
|
||
|
||
# _reset_cffi не должен реально пересоздавать сессию (иначе .get-side_effect
|
||
# потеряется) — патчим на no-op счётчик, проверяем что вызвался.
|
||
reset = AsyncMock()
|
||
with (
|
||
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()),
|
||
patch.object(AvitoScraper, "_reset_cffi", reset),
|
||
patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot,
|
||
):
|
||
html = await scraper._fetch_serp_html("https://www.avito.ru/x", page=1)
|
||
|
||
assert html == _CARD_HTML
|
||
assert reset.await_count == 2 # 403, 403 → два reconnect перед 200
|
||
rot.assert_not_awaited() # backconnect не дёргает changeip-ротацию
|
||
assert mock_session.get.await_count == 3
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_backconnect_firewall_reconnect_succeeds(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""backconnect: firewall-200 → 200 → reconnect-retry (firewall тоже триггерит)."""
|
||
_enable_backconnect(monkeypatch)
|
||
|
||
scraper = AvitoScraper()
|
||
fw = MagicMock()
|
||
fw.status_code = 200
|
||
fw.text = _FIREWALL_HTML
|
||
ok = MagicMock()
|
||
ok.status_code = 200
|
||
ok.text = _CARD_HTML
|
||
mock_session = AsyncMock()
|
||
mock_session.get = AsyncMock(side_effect=[fw, ok])
|
||
scraper._cffi = mock_session
|
||
|
||
reset = AsyncMock()
|
||
with (
|
||
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()),
|
||
patch.object(AvitoScraper, "_reset_cffi", reset),
|
||
):
|
||
html = await scraper._fetch_serp_html("https://www.avito.ru/x", page=1)
|
||
|
||
assert html == _CARD_HTML
|
||
assert reset.await_count == 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_backconnect_403_exhausted_raises_blocked(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""backconnect: 403 _AVITO_403_MAX_RETRIES+1 раз → всё равно AvitoBlockedError."""
|
||
_enable_backconnect(monkeypatch)
|
||
|
||
scraper = AvitoScraper()
|
||
r403 = MagicMock()
|
||
r403.status_code = 403
|
||
r403.text = ""
|
||
mock_session = AsyncMock()
|
||
mock_session.get = AsyncMock(return_value=r403)
|
||
scraper._cffi = mock_session
|
||
|
||
reset = AsyncMock()
|
||
with (
|
||
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()),
|
||
patch.object(AvitoScraper, "_reset_cffi", reset),
|
||
):
|
||
with pytest.raises(AvitoBlockedError, match="IP banned"):
|
||
await scraper._fetch_serp_html("https://www.avito.ru/x", page=1)
|
||
|
||
# ровно _AVITO_403_MAX_RETRIES reconnect, потом raise
|
||
assert reset.await_count == avito_mod._AVITO_403_MAX_RETRIES
|
||
# get: 1 первичный + _AVITO_403_MAX_RETRIES повторов
|
||
assert mock_session.get.await_count == avito_mod._AVITO_403_MAX_RETRIES + 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_reconnect_fires_even_when_rotate_url_set(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""Прод-конфиг run-212 регрессия: scraper_proxy_url + leftover avito_proxy_rotate_url +
|
||
max_rotations=0 → reconnect-retry ВСЁ РАВНО срабатывает (gate только на наличие proxy).
|
||
|
||
changeip-URL относится к ДРУГОМУ (auv/browser) прокси; curl_cffi egress через
|
||
scraper_proxy_url, поэтому пересоздание сессии меняет exit-IP независимо от changeip.
|
||
С max_rotations=0 changeip-путь — no-op, fallback-reconnect снимает 403.
|
||
"""
|
||
monkeypatch.setattr(
|
||
avito_mod.settings, "avito_proxy_rotate_url", "https://changeip.example/?proxy_key=x"
|
||
)
|
||
monkeypatch.setattr(avito_mod.settings, "avito_proxy_max_rotations", 0)
|
||
monkeypatch.setattr(avito_mod.settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619")
|
||
|
||
scraper = AvitoScraper()
|
||
r403 = MagicMock()
|
||
r403.status_code = 403
|
||
r403.text = ""
|
||
ok = MagicMock()
|
||
ok.status_code = 200
|
||
ok.text = _CARD_HTML
|
||
mock_session = AsyncMock()
|
||
mock_session.get = AsyncMock(side_effect=[r403, r403, ok])
|
||
scraper._cffi = mock_session
|
||
|
||
reset = AsyncMock()
|
||
with (
|
||
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()),
|
||
patch.object(AvitoScraper, "_reset_cffi", reset),
|
||
patch.object(AvitoScraper, "_rotate_ip", AsyncMock(return_value=True)) as rot,
|
||
):
|
||
html = await scraper._fetch_serp_html("https://www.avito.ru/x", page=1)
|
||
|
||
assert html == _CARD_HTML # 403→403→200 снят reconnect'ом, нет raise
|
||
assert reset.await_count == 2 # reconnect сработал несмотря на rotate_url
|
||
rot.assert_not_awaited() # max_rotations=0 → changeip-путь не дёргается
|
||
assert mock_session.get.await_count == 3
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_changeip_path_tried_first_when_rotations_available(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""changeip precedence: scraper_proxy_url + rotate_url + max_rotations=1 → на 403
|
||
changeip-ротация пробуется ПЕРЕД reconnect-fallback.
|
||
|
||
Гарантирует, что новый gate не сломал приоритет legacy changeip-пути.
|
||
"""
|
||
monkeypatch.setattr(
|
||
avito_mod.settings, "avito_proxy_rotate_url", "https://changeip.example/?proxy_key=x"
|
||
)
|
||
monkeypatch.setattr(avito_mod.settings, "avito_proxy_max_rotations", 1)
|
||
monkeypatch.setattr(avito_mod.settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619")
|
||
|
||
scraper = AvitoScraper()
|
||
r403 = MagicMock()
|
||
r403.status_code = 403
|
||
r403.text = ""
|
||
ok = MagicMock()
|
||
ok.status_code = 200
|
||
ok.text = _CARD_HTML
|
||
mock_session = AsyncMock()
|
||
mock_session.get = AsyncMock(side_effect=[r403, ok])
|
||
scraper._cffi = mock_session
|
||
|
||
reset = AsyncMock()
|
||
rot = AsyncMock(return_value=True)
|
||
with (
|
||
patch("app.services.scrapers.avito.asyncio.sleep", AsyncMock()),
|
||
patch.object(AvitoScraper, "_reset_cffi", reset),
|
||
patch.object(AvitoScraper, "_rotate_ip", rot),
|
||
):
|
||
html = await scraper._fetch_serp_html("https://www.avito.ru/x", page=1)
|
||
|
||
assert html == _CARD_HTML
|
||
rot.assert_awaited_once() # changeip-путь использован первым на 403
|
||
reset.assert_not_awaited() # до reconnect-fallback не дошло
|
||
assert mock_session.get.await_count == 2
|