109 lines
4.3 KiB
Python
109 lines
4.3 KiB
Python
"""Browser-only режим fetch_detail (#1814).
|
||
|
||
Когда передан browser_fetcher, detail-fetch идёт СТРОГО через camoufox-сервис.
|
||
Никакой curl_cffi-сессии (AsyncSession) и никакого curl-fallback — avito-антибот
|
||
банит curl-фингерпринт под нагрузкой (429 CONNECT tunnel failed). Любой
|
||
firewall/ошибка браузера → AvitoBlockedError; caller (backfill) метит failed и идёт
|
||
дальше, НЕ создавая curl-сессию.
|
||
|
||
Тесты проверяют именно отсутствие инстанцирования AsyncSession и поднятие
|
||
AvitoBlockedError на browser-firewall/ошибке.
|
||
"""
|
||
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import httpx
|
||
import pytest
|
||
|
||
from app.services.scrapers.avito_detail import fetch_detail
|
||
from app.services.scrapers.avito_exceptions import AvitoBlockedError
|
||
|
||
_SENTINEL = object()
|
||
_FIREWALL_HTML = (
|
||
"<html><head><title>Доступ ограничен: проблема с IP</title></head>"
|
||
'<body><div class="firewall-container">...</div></body></html>'
|
||
)
|
||
_OK_HTML = "<html><body>ok</body></html>"
|
||
|
||
# AsyncSession (curl_cffi) в модуле avito_detail — НЕ должна инстанцироваться в browser-mode.
|
||
_ASYNC_SESSION = "app.services.scrapers.avito_detail.AsyncSession"
|
||
_BUILD_SESSION = "app.services.scrapers.avito_detail._build_detail_session"
|
||
_PARSE = "app.services.scrapers.avito_detail.parse_detail_html"
|
||
|
||
|
||
def _browser(*, fetch_side: object) -> MagicMock:
|
||
bf = MagicMock()
|
||
if isinstance(fetch_side, BaseException):
|
||
bf.fetch = AsyncMock(side_effect=fetch_side)
|
||
else:
|
||
bf.fetch = AsyncMock(return_value=fetch_side)
|
||
return bf
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_browser_mode_success_no_curl_session() -> None:
|
||
"""browser_fetcher OK → parse, AsyncSession НЕ создаётся."""
|
||
bf = _browser(fetch_side=_OK_HTML)
|
||
with (
|
||
patch(_ASYNC_SESSION) as async_session,
|
||
patch(_BUILD_SESSION) as build_session,
|
||
patch(_PARSE, return_value=_SENTINEL) as parse,
|
||
):
|
||
result = await fetch_detail("/items/123", browser_fetcher=bf)
|
||
|
||
assert result is _SENTINEL
|
||
bf.fetch.assert_awaited_once()
|
||
parse.assert_called_once()
|
||
async_session.assert_not_called()
|
||
build_session.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_browser_firewall_raises_blocked_no_curl_session() -> None:
|
||
"""browser вернул firewall-HTML (status 200) → AvitoBlockedError, без curl."""
|
||
bf = _browser(fetch_side=_FIREWALL_HTML)
|
||
with (
|
||
patch(_ASYNC_SESSION) as async_session,
|
||
patch(_BUILD_SESSION) as build_session,
|
||
patch(_PARSE, return_value=_SENTINEL),
|
||
):
|
||
with pytest.raises(AvitoBlockedError):
|
||
await fetch_detail("/items/123", browser_fetcher=bf)
|
||
|
||
async_session.assert_not_called()
|
||
build_session.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_browser_http_error_raises_blocked_no_curl_session() -> None:
|
||
"""browser_fetcher.fetch кидает HTTPError (502/firewall сервиса) → AvitoBlockedError,
|
||
AsyncSession НЕ инстанцируется (НЕ откатываемся в curl-фингерпринт → 429-бан)."""
|
||
bf = _browser(fetch_side=httpx.HTTPError("browser service 502"))
|
||
with (
|
||
patch(_ASYNC_SESSION) as async_session,
|
||
patch(_BUILD_SESSION) as build_session,
|
||
patch(_PARSE, return_value=_SENTINEL),
|
||
):
|
||
with pytest.raises(AvitoBlockedError):
|
||
await fetch_detail("/items/123", browser_fetcher=bf)
|
||
|
||
async_session.assert_not_called()
|
||
build_session.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_browser_blocked_error_propagates_unchanged() -> None:
|
||
"""Если browser_fetcher сам кинул AvitoBlockedError — пробрасываем как есть, без curl."""
|
||
original = AvitoBlockedError("already blocked")
|
||
bf = _browser(fetch_side=original)
|
||
with (
|
||
patch(_ASYNC_SESSION) as async_session,
|
||
patch(_BUILD_SESSION) as build_session,
|
||
patch(_PARSE, return_value=_SENTINEL),
|
||
):
|
||
with pytest.raises(AvitoBlockedError) as exc_info:
|
||
await fetch_detail("/items/123", browser_fetcher=bf)
|
||
|
||
assert exc_info.value is original
|
||
async_session.assert_not_called()
|
||
build_session.assert_not_called()
|