bot-backend
5ed88c41a4
Some checks failed
Deploy Trade-In / build-backend (push) Blocked by required conditions
Deploy Trade-In / deploy (push) Blocked by required conditions
Deploy Trade-In / changes (push) Successful in 10s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Has been cancelled
avito SERP browser post-merge polish (#1811): unify fetch_mode guard, empty-render guard, debug logging. Refs #1811
520 lines
22 KiB
Python
520 lines
22 KiB
Python
"""#901/#1805 — avito SERP fetch via BrowserFetcher (camoufox), curl_cffi fallback.
|
||
|
||
Tests:
|
||
1. browser branch default (#1805): __aenter__ → _browser set AND _cffi fallback armed.
|
||
2. curl branch unchanged: _browser is None → _cffi session used directly.
|
||
3. firewall-detect in browser-mode: firewall HTML + no rotate URL + no cffi → AvitoBlockedError.
|
||
3b. #1805 curl_cffi fallback: browser firewall → curl retry returns HTML / re-raises if also banned.
|
||
4. pagination terminates via empty-parse: browser HTML with 0 lots → caller loop breaks.
|
||
|
||
Без сети, без БД, без curl_cffi.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import httpx
|
||
import pytest
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||
|
||
from app.services.scrapers.avito import _FIREWALL_MARKERS, AvitoScraper
|
||
from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
|
||
from app.services.scrapers.base import ScrapedLot
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_NORMAL_HTML = "<html><body>normal avito SERP</body></html>"
|
||
_FIREWALL_HTML = f"<html><head><title>{_FIREWALL_MARKERS[0]}</title></head></html>"
|
||
|
||
|
||
def _sidecar_error(status: int, error: str) -> httpx.HTTPStatusError:
|
||
"""Build a tradein-browser sidecar HTTPStatusError with a real JSON body.
|
||
|
||
Mirrors what BrowserFetcher.fetch re-raises: an httpx.HTTPStatusError whose
|
||
`.response` is intact so `.response.json()["error"]` and `.response.status_code`
|
||
work the same as in production.
|
||
"""
|
||
request = httpx.Request("POST", "http://tradein-browser:3000/fetch")
|
||
response = httpx.Response(status, json={"error": error}, request=request)
|
||
return httpx.HTTPStatusError(f"{status} error", request=request, response=response)
|
||
|
||
|
||
def _make_mock_browser(html: str = _NORMAL_HTML) -> MagicMock:
|
||
"""Return a mock BrowserFetcher context manager that yields html from fetch()."""
|
||
mock = MagicMock()
|
||
mock.__aenter__ = AsyncMock(return_value=mock)
|
||
mock.__aexit__ = AsyncMock(return_value=None)
|
||
mock.fetch = AsyncMock(return_value=html)
|
||
return mock
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 1. browser branch chosen
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_aenter_browser_mode_arms_cffi_fallback() -> None:
|
||
"""#1805: mode=browser → __aenter__ поднимает BrowserFetcher И curl_cffi-fallback.
|
||
|
||
Раньше browser-mode оставлял _cffi=None; теперь curl_cffi-сессия строится тут
|
||
же как fallback на firewall/пусто (см. _fetch_serp_html_browser).
|
||
"""
|
||
mock_browser = _make_mock_browser()
|
||
scraper = AvitoScraper()
|
||
|
||
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||
mock_settings.scraper_fetch_mode = "browser"
|
||
mock_settings.scraper_proxy_url = None
|
||
mock_settings.avito_proxy_rotate_url = None
|
||
mock_settings.avito_proxy_max_rotations = 2
|
||
|
||
with patch(
|
||
"app.services.scrapers.avito.BrowserFetcher",
|
||
return_value=mock_browser,
|
||
):
|
||
result = await scraper.__aenter__()
|
||
|
||
assert result is scraper
|
||
assert scraper._browser is mock_browser
|
||
# curl_cffi fallback armed (real AsyncSession built, no network/proxy).
|
||
assert scraper._cffi is not None
|
||
await scraper._cffi.close()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_aenter_proactive_rotate_when_rotate_url_set() -> None:
|
||
"""#1731: rotate_url configured → __aenter__ calls _rotate_ip exactly once at start.
|
||
|
||
_rotate_ip is mocked (no real changeip HTTP / no 9s sleep). Browser mode used
|
||
to keep BaseScraper setup out of the picture.
|
||
"""
|
||
mock_browser = _make_mock_browser()
|
||
scraper = AvitoScraper()
|
||
mock_rotate = AsyncMock(return_value=True)
|
||
|
||
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||
mock_settings.scraper_fetch_mode = "browser"
|
||
mock_settings.scraper_proxy_url = None
|
||
mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/changeip"
|
||
mock_settings.avito_proxy_max_rotations = 4
|
||
|
||
with patch(
|
||
"app.services.scrapers.avito.BrowserFetcher",
|
||
return_value=mock_browser,
|
||
):
|
||
with patch.object(scraper, "_rotate_ip", new=mock_rotate):
|
||
result = await scraper.__aenter__()
|
||
|
||
assert result is scraper
|
||
mock_rotate.assert_awaited_once()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_aenter_no_proactive_rotate_when_rotate_url_none() -> None:
|
||
"""#1731: rotate_url is None → __aenter__ must NOT call _rotate_ip."""
|
||
mock_browser = _make_mock_browser()
|
||
scraper = AvitoScraper()
|
||
mock_rotate = AsyncMock(return_value=True)
|
||
|
||
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||
mock_settings.scraper_fetch_mode = "browser"
|
||
mock_settings.scraper_proxy_url = None
|
||
mock_settings.avito_proxy_rotate_url = None
|
||
mock_settings.avito_proxy_max_rotations = 4
|
||
|
||
with patch(
|
||
"app.services.scrapers.avito.BrowserFetcher",
|
||
return_value=mock_browser,
|
||
):
|
||
with patch.object(scraper, "_rotate_ip", new=mock_rotate):
|
||
await scraper.__aenter__()
|
||
|
||
mock_rotate.assert_not_awaited()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_aenter_proactive_rotate_failure_does_not_abort() -> None:
|
||
"""#1731: a failed proactive rotate (returns False / raises) must NOT abort __aenter__."""
|
||
mock_browser = _make_mock_browser()
|
||
scraper = AvitoScraper()
|
||
|
||
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||
mock_settings.scraper_fetch_mode = "browser"
|
||
mock_settings.scraper_proxy_url = None
|
||
mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/changeip"
|
||
mock_settings.avito_proxy_max_rotations = 4
|
||
|
||
with patch(
|
||
"app.services.scrapers.avito.BrowserFetcher",
|
||
return_value=mock_browser,
|
||
):
|
||
# _rotate_ip raises — sweep must proceed (browser still attached)
|
||
with patch.object(
|
||
scraper, "_rotate_ip", new=AsyncMock(side_effect=RuntimeError("changeip down"))
|
||
):
|
||
result = await scraper.__aenter__()
|
||
|
||
assert result is scraper
|
||
assert scraper._browser is mock_browser
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_serp_html_browser_mode_returns_html() -> None:
|
||
"""_fetch_serp_html with _browser set returns HTML string, never touches _cffi."""
|
||
scraper = AvitoScraper()
|
||
mock_browser = _make_mock_browser(_NORMAL_HTML)
|
||
scraper._browser = mock_browser
|
||
|
||
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
|
||
|
||
assert result == _NORMAL_HTML
|
||
mock_browser.fetch.assert_awaited_once_with("https://www.avito.ru/test")
|
||
# _cffi not armed (direct _browser assignment bypasses __aenter__, which would
|
||
# otherwise build the curl_cffi fallback session); the browser path never touches it.
|
||
assert scraper._cffi is None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 2. curl branch unchanged
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_serp_html_curl_mode_uses_cffi() -> None:
|
||
"""Default mode (_browser is None) → _fetch_serp_html goes through curl path."""
|
||
scraper = AvitoScraper()
|
||
|
||
# Mock the curl session returning HTTP 200 with normal HTML
|
||
mock_response = MagicMock()
|
||
mock_response.status_code = 200
|
||
mock_response.text = _NORMAL_HTML
|
||
|
||
mock_cffi = MagicMock()
|
||
mock_cffi.get = AsyncMock(return_value=mock_response)
|
||
scraper._cffi = mock_cffi
|
||
|
||
# _browser must be None (default curl path)
|
||
assert scraper._browser is None
|
||
|
||
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
|
||
|
||
assert result == _NORMAL_HTML
|
||
mock_cffi.get.assert_awaited_once_with("https://www.avito.ru/test")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 3. firewall-detect in browser-mode → AvitoBlockedError (no rotate URL)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_browser_mode_firewall_raises_blocked_error() -> None:
|
||
"""Browser firewall + no rotate URL + no curl fallback (_cffi=None) → AvitoBlockedError."""
|
||
scraper = AvitoScraper()
|
||
mock_browser = _make_mock_browser(_FIREWALL_HTML)
|
||
scraper._browser = mock_browser
|
||
scraper._cffi = None # no curl_cffi fallback available (pipeline shared-browser path)
|
||
|
||
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||
mock_settings.avito_proxy_rotate_url = None # no rotation configured
|
||
mock_settings.avito_proxy_max_rotations = 2
|
||
|
||
with pytest.raises(AvitoBlockedError, match="browser-mode"):
|
||
await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 3b. #1805 — curl_cffi fallback when browser returns firewall
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_browser_firewall_falls_back_to_cffi_returns_html() -> None:
|
||
"""#1805: browser firewall + curl_cffi fallback armed → curl returns clean HTML."""
|
||
scraper = AvitoScraper()
|
||
scraper._browser = _make_mock_browser(_FIREWALL_HTML)
|
||
|
||
# curl_cffi fallback: HTTP 200 with clean (non-firewall) HTML.
|
||
mock_resp = MagicMock()
|
||
mock_resp.status_code = 200
|
||
mock_resp.text = _NORMAL_HTML
|
||
mock_cffi = MagicMock()
|
||
mock_cffi.get = AsyncMock(return_value=mock_resp)
|
||
scraper._cffi = mock_cffi
|
||
|
||
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||
mock_settings.avito_proxy_rotate_url = None
|
||
mock_settings.avito_proxy_max_rotations = 0
|
||
mock_settings.scraper_proxy_url = None
|
||
|
||
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
|
||
|
||
assert result == _NORMAL_HTML
|
||
mock_cffi.get.assert_awaited_once_with("https://www.avito.ru/test")
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_browser_firewall_cffi_fallback_also_firewall_raises() -> None:
|
||
"""#1805: browser firewall AND curl_cffi fallback also firewall → AvitoBlockedError."""
|
||
scraper = AvitoScraper()
|
||
scraper._browser = _make_mock_browser(_FIREWALL_HTML)
|
||
|
||
# curl_cffi fallback: HTTP 200 but ALSO firewall HTML → no escape.
|
||
mock_resp = MagicMock()
|
||
mock_resp.status_code = 200
|
||
mock_resp.text = _FIREWALL_HTML
|
||
mock_cffi = MagicMock()
|
||
mock_cffi.get = AsyncMock(return_value=mock_resp)
|
||
scraper._cffi = mock_cffi
|
||
|
||
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||
mock_settings.avito_proxy_rotate_url = None
|
||
mock_settings.avito_proxy_max_rotations = 0
|
||
mock_settings.scraper_proxy_url = None # no backconnect reconnect retries
|
||
|
||
# curl fallback raises its own AvitoBlockedError (firewall на HTTP 200);
|
||
# browser branch re-raises it as a hard-block.
|
||
with pytest.raises(AvitoBlockedError):
|
||
await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_browser_mode_firewall_rotates_and_retries() -> None:
|
||
"""Browser returns firewall on first call, normal HTML after rotation → returns HTML."""
|
||
scraper = AvitoScraper()
|
||
|
||
# First call → firewall; second call → normal HTML
|
||
mock_browser = MagicMock()
|
||
mock_browser.fetch = AsyncMock(side_effect=[_FIREWALL_HTML, _NORMAL_HTML])
|
||
scraper._browser = mock_browser
|
||
|
||
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||
mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/rotate"
|
||
mock_settings.avito_proxy_max_rotations = 1
|
||
|
||
with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=True)):
|
||
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=2)
|
||
|
||
assert result == _NORMAL_HTML
|
||
assert mock_browser.fetch.await_count == 2
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 4. pagination terminates via empty-parse
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_city_wide_browser_mode_stops_on_empty_parse() -> None:
|
||
"""fetch_city_wide with browser mode: page=1 returns lots, page=2 returns [] → stops.
|
||
|
||
Verifies the documented invariant: callers break on `not lots` regardless of
|
||
fetch mode. _fetch_serp_html is mocked at method level (browser vs curl is
|
||
already covered by unit tests above); here we confirm the caller loop behaviour.
|
||
"""
|
||
scraper = AvitoScraper()
|
||
|
||
html_p1 = "<html>page1</html>"
|
||
html_p2 = "<html>page2</html>"
|
||
|
||
lot = ScrapedLot(
|
||
source="avito",
|
||
source_url="https://www.avito.ru/ekaterinburg/kvartiry/test",
|
||
source_id="LOT_1",
|
||
price_rub=5_000_000,
|
||
)
|
||
|
||
call_count = 0
|
||
|
||
async def mock_fetch_html(url: str, page: int) -> str:
|
||
return html_p1 if page == 1 else html_p2
|
||
|
||
def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
nonlocal call_count
|
||
call_count += 1
|
||
return [lot] if "page1" in html else []
|
||
|
||
with patch.object(scraper, "_fetch_serp_html", side_effect=mock_fetch_html):
|
||
with patch.object(scraper, "_parse_html", side_effect=mock_parse):
|
||
with patch.object(scraper, "sleep_between_requests", new=AsyncMock()):
|
||
result = await scraper.fetch_city_wide(pages=10, delay_override_sec=0)
|
||
|
||
assert len(result) == 1
|
||
assert result[0].source_id == "LOT_1"
|
||
# page1 parsed (returned lot) + page2 parsed (returned []) → stopped; page3-10 never called
|
||
assert call_count == 2
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 5. rotation returns False → immediate AvitoBlockedError
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_browser_mode_rotation_returns_false_raises_blocked() -> None:
|
||
"""rotate_url set but _rotate_ip() returns False → must immediately raise AvitoBlockedError.
|
||
|
||
Covers the branch: firewall detected + rotate_url configured + _rotate_ip() == False
|
||
(e.g. rotation API timeout/failure) → we cannot recover → blocked.
|
||
"""
|
||
scraper = AvitoScraper()
|
||
scraper._browser = _make_mock_browser(_FIREWALL_HTML)
|
||
|
||
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||
mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
|
||
mock_settings.avito_proxy_max_rotations = 2
|
||
with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=False)):
|
||
with pytest.raises(AvitoBlockedError, match="browser-mode"):
|
||
await scraper._fetch_serp_html("https://avito.ru/test", page=1)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 6. #356 — sidecar HTTPStatusError (500/503) → graceful AvitoRateLimitedError
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sidecar_proxy_too_many_requests_rotates_then_succeeds() -> None:
|
||
"""#356: sidecar 500 NS_ERROR_PROXY_TOO_MANY_REQUESTS, rotation available →
|
||
after IP rotation a retry succeeds and the clean HTML is returned."""
|
||
scraper = AvitoScraper()
|
||
|
||
mock_browser = MagicMock()
|
||
# First fetch raises the soft-ban 500; after rotation the second fetch is clean.
|
||
mock_browser.fetch = AsyncMock(
|
||
side_effect=[
|
||
_sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"),
|
||
_NORMAL_HTML,
|
||
]
|
||
)
|
||
scraper._browser = mock_browser
|
||
mock_rotate = AsyncMock(return_value=True)
|
||
|
||
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||
mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
|
||
mock_settings.avito_proxy_max_rotations = 2
|
||
with patch.object(scraper, "_rotate_ip", new=mock_rotate):
|
||
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=3)
|
||
|
||
assert result == _NORMAL_HTML
|
||
mock_rotate.assert_awaited_once()
|
||
assert mock_browser.fetch.await_count == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sidecar_proxy_too_many_requests_exhausted_raises_rate_limited() -> None:
|
||
"""#356: sidecar 500 proxy-too-many with rotation budget exhausted →
|
||
AvitoRateLimitedError (NOT a bare HTTPStatusError, NOT AvitoBlockedError)."""
|
||
scraper = AvitoScraper()
|
||
|
||
mock_browser = MagicMock()
|
||
mock_browser.fetch = AsyncMock(
|
||
side_effect=_sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS")
|
||
)
|
||
scraper._browser = mock_browser
|
||
mock_rotate = AsyncMock(return_value=True)
|
||
|
||
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||
mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
|
||
mock_settings.avito_proxy_max_rotations = 2
|
||
with patch.object(scraper, "_rotate_ip", new=mock_rotate):
|
||
with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
|
||
await scraper._fetch_serp_html("https://www.avito.ru/test", page=3)
|
||
|
||
# 2 rotations consumed + 1 final fetch that exhausts the budget.
|
||
assert mock_rotate.await_count == 2
|
||
assert mock_browser.fetch.await_count == 3
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sidecar_transient_timeout_exhausts_budget_no_infinite_loop() -> None:
|
||
"""#356: sidecar 500 transient (Timeout) repeated past the transient budget with
|
||
NO rotations → AvitoRateLimitedError after bounded retries (no infinite loop)."""
|
||
scraper = AvitoScraper()
|
||
|
||
mock_browser = MagicMock()
|
||
mock_browser.fetch = AsyncMock(
|
||
side_effect=_sidecar_error(500, "TimeoutError: Page.goto: Timeout 60000ms exceeded")
|
||
)
|
||
scraper._browser = mock_browser
|
||
mock_rotate = AsyncMock(return_value=True)
|
||
|
||
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||
mock_settings.avito_proxy_rotate_url = None # no rotation → transient budget only
|
||
mock_settings.avito_proxy_max_rotations = 0
|
||
# Patch sleep so the bounded backoff doesn't slow the test.
|
||
with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
|
||
with patch.object(scraper, "_rotate_ip", new=mock_rotate):
|
||
with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
|
||
await scraper._fetch_serp_html("https://www.avito.ru/test", page=5)
|
||
|
||
# Transient is NOT a soft-ban → never rotates.
|
||
mock_rotate.assert_not_awaited()
|
||
# Bounded: _AVITO_SIDECAR_TRANSIENT_RETRIES (2) retries + final attempt = 3.
|
||
assert mock_browser.fetch.await_count == 3
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sidecar_503_browser_unavailable_exhausted_raises_rate_limited() -> None:
|
||
"""#356: sidecar 503 'browser unavailable (proxy may be down)' with rotation
|
||
exhausted → AvitoRateLimitedError (soft-ban classification via status==503)."""
|
||
scraper = AvitoScraper()
|
||
|
||
mock_browser = MagicMock()
|
||
mock_browser.fetch = AsyncMock(
|
||
side_effect=_sidecar_error(503, "browser unavailable (proxy may be down)")
|
||
)
|
||
scraper._browser = mock_browser
|
||
mock_rotate = AsyncMock(return_value=True)
|
||
|
||
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||
mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
|
||
mock_settings.avito_proxy_max_rotations = 1
|
||
with patch.object(scraper, "_rotate_ip", new=mock_rotate):
|
||
with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
|
||
await scraper._fetch_serp_html("https://www.avito.ru/test", page=7)
|
||
|
||
# 1 rotation consumed then exhausted.
|
||
assert mock_rotate.await_count == 1
|
||
assert mock_browser.fetch.await_count == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sidecar_transient_then_soft_ban_independent_budgets() -> None:
|
||
"""#356: смешанная последовательность transient → soft-ban пинит, что бюджеты
|
||
независимы и корректно чередуются в одном while-цикле.
|
||
|
||
attempt 1: transient Timeout → тратит transient_left (1 из 2), backoff, continue.
|
||
attempt 2: too_many_requests → soft-ban, тратит ротацию (1 из 1), continue.
|
||
attempt 3: too_many_requests, ротация исчерпана → AvitoRateLimitedError.
|
||
"""
|
||
scraper = AvitoScraper()
|
||
|
||
mock_browser = MagicMock()
|
||
mock_browser.fetch = AsyncMock(
|
||
side_effect=[
|
||
_sidecar_error(500, "TimeoutError: Page.goto: Timeout 60000ms exceeded"),
|
||
_sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"),
|
||
_sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"),
|
||
]
|
||
)
|
||
scraper._browser = mock_browser
|
||
mock_rotate = AsyncMock(return_value=True)
|
||
|
||
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||
mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
|
||
mock_settings.avito_proxy_max_rotations = 1
|
||
with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
|
||
with patch.object(scraper, "_rotate_ip", new=mock_rotate):
|
||
with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
|
||
await scraper._fetch_serp_html("https://www.avito.ru/test", page=9)
|
||
|
||
# 1 transient retry + 1 rotation, оба бюджета тронуты до исчерпания.
|
||
assert mock_rotate.await_count == 1
|
||
assert mock_browser.fetch.await_count == 3
|