bot-backend
c993243bc7
fetch_detail raised on the first 403/429, aborting the detail backfill (run 191 enriched only 48 then zombied) -> avito detail coverage stuck at 14% of active. Mirror the merged SERP fix (#1765): on 403/firewall under the backconnect proxy, rebuild an ephemeral session (fresh exit IP) and retry up to N before raising; short-retry 429. Never close/rebuild the caller-provided shared session. Also add the missing HTTP-200 firewall-interstitial check on the curl detail path.
191 lines
7.6 KiB
Python
191 lines
7.6 KiB
Python
"""curl_cffi 403/firewall reconnect-retry на detail-странице под backconnect-прокси.
|
||
|
||
fetch_detail ронял весь detail-backfill на первом 403/429 через AvitoBlockedError /
|
||
AvitoRateLimitedError (run 191 обогатил 48 → zombie). Зеркало SERP-фикса (#1765): под
|
||
backconnect rotating-прокси (mproxy.site) каждое НОВОЕ соединение даёт свежий exit-IP,
|
||
поэтому 403/firewall ретраим пересозданием эфемерной curl_cffi-сессии до
|
||
_AVITO_DETAIL_403_MAX_RETRIES раз. 429 — короткий retry той же сессии без reconnect.
|
||
|
||
Ownership-правило: caller-provided cffi_session НИКОГДА не закрывается/не пересоздаётся —
|
||
retries идут через свою эфемерную retry_session (закрываем в finally).
|
||
|
||
Включаем backconnect через scraper_proxy_url_env (scraper_proxy_url — read-only property,
|
||
его не monkeypatch'ишь). Зеркало fixture-стиля test_avito_403_backconnect_reconnect.py.
|
||
"""
|
||
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from app.core.config import settings
|
||
from app.services.scrapers import avito_detail as detail_mod
|
||
from app.services.scrapers.avito_detail import fetch_detail
|
||
from app.services.scrapers.avito_exceptions import AvitoBlockedError
|
||
|
||
_SENTINEL = object()
|
||
_FIREWALL_HTML = (
|
||
"<html><head><title>Доступ ограничен: проблема с IP</title></head>"
|
||
'<body><div class="firewall-container">...</div></body></html>'
|
||
)
|
||
|
||
|
||
def _resp(status_code: int, text: str = "") -> MagicMock:
|
||
r = MagicMock()
|
||
r.status_code = status_code
|
||
r.text = text
|
||
return r
|
||
|
||
|
||
def _enable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
monkeypatch.setattr(settings, "scraper_proxy_url_env", "http://u:p@mproxy.site:14619")
|
||
|
||
|
||
def _disable_backconnect(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
monkeypatch.setattr(settings, "scraper_proxy_url_env", None)
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _patch_common() -> object:
|
||
"""asyncio.sleep → no-op; parse_detail_html → sentinel (нет нужды в реальном HTML)."""
|
||
with (
|
||
patch("app.services.scrapers.avito_detail.asyncio.sleep", AsyncMock()),
|
||
patch(
|
||
"app.services.scrapers.avito_detail.parse_detail_html", return_value=_SENTINEL
|
||
) as parse,
|
||
):
|
||
yield parse
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_403_403_200_caller_session_reconnect_succeeds(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""caller-provided session: 403 → 403 → 200 → reconnect-retry возвращает enrichment.
|
||
|
||
_build_detail_session дёрнут для retries; caller-session НЕ закрыта.
|
||
"""
|
||
_enable_backconnect(monkeypatch)
|
||
|
||
caller = AsyncMock()
|
||
caller.get = AsyncMock(return_value=_resp(403))
|
||
|
||
# Каждый retry → свежая эфемерная сессия. Первая отдаёт 403 (для 2-го reconnect),
|
||
# вторая — 200.
|
||
retry1 = AsyncMock()
|
||
retry1.get = AsyncMock(return_value=_resp(403))
|
||
retry2 = AsyncMock()
|
||
retry2.get = AsyncMock(return_value=_resp(200, "<html/>"))
|
||
build = MagicMock(side_effect=[retry1, retry2])
|
||
|
||
with patch.object(detail_mod, "_build_detail_session", build):
|
||
result = await fetch_detail("/ekaterinburg/kvartiry/test-123", cffi_session=caller)
|
||
|
||
assert result is _SENTINEL
|
||
assert build.call_count == 2 # 403, 403 → два reconnect перед 200
|
||
caller.close.assert_not_called() # caller-provided session НЕ закрывается
|
||
assert caller.get.await_count == 1 # первый attempt
|
||
retry1.close.assert_awaited() # промежуточная retry-session закрыта
|
||
retry2.close.assert_awaited() # последняя retry-session закрыта в finally
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_firewall_200_reconnect_succeeds(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""firewall(HTTP 200) → 200 → новый HTTP-200 firewall-detect триггерит reconnect."""
|
||
_enable_backconnect(monkeypatch)
|
||
|
||
caller = AsyncMock()
|
||
caller.get = AsyncMock(return_value=_resp(200, _FIREWALL_HTML))
|
||
retry1 = AsyncMock()
|
||
retry1.get = AsyncMock(return_value=_resp(200, "<html/>"))
|
||
build = MagicMock(side_effect=[retry1])
|
||
|
||
with patch.object(detail_mod, "_build_detail_session", build):
|
||
result = await fetch_detail("/x", cffi_session=caller)
|
||
|
||
assert result is _SENTINEL
|
||
assert build.call_count == 1
|
||
caller.close.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_403_exhausted_raises_blocked(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""403 _AVITO_DETAIL_403_MAX_RETRIES+1 раз → AvitoBlockedError."""
|
||
_enable_backconnect(monkeypatch)
|
||
|
||
caller = AsyncMock()
|
||
caller.get = AsyncMock(return_value=_resp(403))
|
||
|
||
def _make_retry() -> AsyncMock:
|
||
s = AsyncMock()
|
||
s.get = AsyncMock(return_value=_resp(403))
|
||
return s
|
||
|
||
build = MagicMock(side_effect=lambda: _make_retry())
|
||
|
||
with patch.object(detail_mod, "_build_detail_session", build):
|
||
with pytest.raises(AvitoBlockedError, match="HTTP 403"):
|
||
await fetch_detail("/x", cffi_session=caller)
|
||
|
||
assert build.call_count == detail_mod._AVITO_DETAIL_403_MAX_RETRIES
|
||
caller.close.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_429_short_retry_then_200_same_session(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""429 → 429 → 200: short-retry той же сессии, без reconnect."""
|
||
_enable_backconnect(monkeypatch)
|
||
|
||
caller = AsyncMock()
|
||
caller.get = AsyncMock(side_effect=[_resp(429), _resp(429), _resp(200, "<html/>")])
|
||
build = MagicMock()
|
||
|
||
with patch.object(detail_mod, "_build_detail_session", build):
|
||
result = await fetch_detail("/x", cffi_session=caller)
|
||
|
||
assert result is _SENTINEL
|
||
build.assert_not_called() # 429 не дёргает reconnect
|
||
assert caller.get.await_count == 3 # та же сессия переиспользована
|
||
caller.close.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_no_proxy_single_403_raises_immediately(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""scraper_proxy_url unset → нет reconnect-пути → первый 403 сразу raise."""
|
||
_disable_backconnect(monkeypatch)
|
||
|
||
caller = AsyncMock()
|
||
caller.get = AsyncMock(return_value=_resp(403))
|
||
build = MagicMock()
|
||
|
||
with patch.object(detail_mod, "_build_detail_session", build):
|
||
with pytest.raises(AvitoBlockedError, match="HTTP 403"):
|
||
await fetch_detail("/x", cffi_session=caller)
|
||
|
||
build.assert_not_called()
|
||
assert caller.get.await_count == 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_own_session_403_200_closes_both(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""own-session path (cffi_session=None): 403 → 200; own session + retry session
|
||
обе закрыты в finally."""
|
||
_enable_backconnect(monkeypatch)
|
||
|
||
own = AsyncMock()
|
||
own.get = AsyncMock(return_value=_resp(403))
|
||
retry1 = AsyncMock()
|
||
retry1.get = AsyncMock(return_value=_resp(200, "<html/>"))
|
||
# Первый _build_detail_session → own (own_session-ветка), второй → retry1 (reconnect).
|
||
build = MagicMock(side_effect=[own, retry1])
|
||
|
||
with patch.object(detail_mod, "_build_detail_session", build):
|
||
result = await fetch_detail("/x", cffi_session=None)
|
||
|
||
assert result is _SENTINEL
|
||
own.close.assert_awaited() # own session закрыта (own_session=True)
|
||
retry1.close.assert_awaited() # retry session закрыта в finally
|
||
assert build.call_count == 2
|