360 lines
14 KiB
Python
360 lines
14 KiB
Python
"""#901/#1805 — avito SERP fetch via BrowserFetcher (camoufox), curl_cffi fallback.
|
|
|
|
Tests:
|
|
1. browser branch default (#1805): __aenter__ → _browser set AND _cffi fallback armed.
|
|
2. curl branch unchanged: _browser is None → _cffi session used directly.
|
|
3. firewall-detect in browser-mode: firewall HTML + no rotate URL + no cffi → AvitoBlockedError.
|
|
3b. #1805 curl_cffi fallback: browser firewall → curl retry returns HTML / re-raises if also banned.
|
|
4. pagination terminates via empty-parse: browser HTML with 0 lots → caller loop breaks.
|
|
|
|
Без сети, без БД, без curl_cffi.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
|
|
|
from app.services.scrapers.avito import _FIREWALL_MARKERS, AvitoScraper
|
|
from app.services.scrapers.avito_exceptions import AvitoBlockedError
|
|
from app.services.scrapers.base import ScrapedLot
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_NORMAL_HTML = "<html><body>normal avito SERP</body></html>"
|
|
_FIREWALL_HTML = f"<html><head><title>{_FIREWALL_MARKERS[0]}</title></head></html>"
|
|
|
|
|
|
def _make_mock_browser(html: str = _NORMAL_HTML) -> MagicMock:
|
|
"""Return a mock BrowserFetcher context manager that yields html from fetch()."""
|
|
mock = MagicMock()
|
|
mock.__aenter__ = AsyncMock(return_value=mock)
|
|
mock.__aexit__ = AsyncMock(return_value=None)
|
|
mock.fetch = AsyncMock(return_value=html)
|
|
return mock
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 1. browser branch chosen
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_aenter_browser_mode_arms_cffi_fallback() -> None:
|
|
"""#1805: mode=browser → __aenter__ поднимает BrowserFetcher И curl_cffi-fallback.
|
|
|
|
Раньше browser-mode оставлял _cffi=None; теперь curl_cffi-сессия строится тут
|
|
же как fallback на firewall/пусто (см. _fetch_serp_html_browser).
|
|
"""
|
|
mock_browser = _make_mock_browser()
|
|
scraper = AvitoScraper()
|
|
|
|
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
|
mock_settings.scraper_fetch_mode = "browser"
|
|
mock_settings.scraper_proxy_url = None
|
|
mock_settings.avito_proxy_rotate_url = None
|
|
mock_settings.avito_proxy_max_rotations = 2
|
|
|
|
with patch(
|
|
"app.services.scrapers.avito.BrowserFetcher",
|
|
return_value=mock_browser,
|
|
):
|
|
result = await scraper.__aenter__()
|
|
|
|
assert result is scraper
|
|
assert scraper._browser is mock_browser
|
|
# curl_cffi fallback armed (real AsyncSession built, no network/proxy).
|
|
assert scraper._cffi is not None
|
|
await scraper._cffi.close()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_aenter_proactive_rotate_when_rotate_url_set() -> None:
|
|
"""#1731: rotate_url configured → __aenter__ calls _rotate_ip exactly once at start.
|
|
|
|
_rotate_ip is mocked (no real changeip HTTP / no 9s sleep). Browser mode used
|
|
to keep BaseScraper setup out of the picture.
|
|
"""
|
|
mock_browser = _make_mock_browser()
|
|
scraper = AvitoScraper()
|
|
mock_rotate = AsyncMock(return_value=True)
|
|
|
|
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
|
mock_settings.scraper_fetch_mode = "browser"
|
|
mock_settings.scraper_proxy_url = None
|
|
mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/changeip"
|
|
mock_settings.avito_proxy_max_rotations = 4
|
|
|
|
with patch(
|
|
"app.services.scrapers.avito.BrowserFetcher",
|
|
return_value=mock_browser,
|
|
):
|
|
with patch.object(scraper, "_rotate_ip", new=mock_rotate):
|
|
result = await scraper.__aenter__()
|
|
|
|
assert result is scraper
|
|
mock_rotate.assert_awaited_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_aenter_no_proactive_rotate_when_rotate_url_none() -> None:
|
|
"""#1731: rotate_url is None → __aenter__ must NOT call _rotate_ip."""
|
|
mock_browser = _make_mock_browser()
|
|
scraper = AvitoScraper()
|
|
mock_rotate = AsyncMock(return_value=True)
|
|
|
|
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
|
mock_settings.scraper_fetch_mode = "browser"
|
|
mock_settings.scraper_proxy_url = None
|
|
mock_settings.avito_proxy_rotate_url = None
|
|
mock_settings.avito_proxy_max_rotations = 4
|
|
|
|
with patch(
|
|
"app.services.scrapers.avito.BrowserFetcher",
|
|
return_value=mock_browser,
|
|
):
|
|
with patch.object(scraper, "_rotate_ip", new=mock_rotate):
|
|
await scraper.__aenter__()
|
|
|
|
mock_rotate.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_aenter_proactive_rotate_failure_does_not_abort() -> None:
|
|
"""#1731: a failed proactive rotate (returns False / raises) must NOT abort __aenter__."""
|
|
mock_browser = _make_mock_browser()
|
|
scraper = AvitoScraper()
|
|
|
|
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
|
mock_settings.scraper_fetch_mode = "browser"
|
|
mock_settings.scraper_proxy_url = None
|
|
mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/changeip"
|
|
mock_settings.avito_proxy_max_rotations = 4
|
|
|
|
with patch(
|
|
"app.services.scrapers.avito.BrowserFetcher",
|
|
return_value=mock_browser,
|
|
):
|
|
# _rotate_ip raises — sweep must proceed (browser still attached)
|
|
with patch.object(
|
|
scraper, "_rotate_ip", new=AsyncMock(side_effect=RuntimeError("changeip down"))
|
|
):
|
|
result = await scraper.__aenter__()
|
|
|
|
assert result is scraper
|
|
assert scraper._browser is mock_browser
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_serp_html_browser_mode_returns_html() -> None:
|
|
"""_fetch_serp_html with _browser set returns HTML string, never touches _cffi."""
|
|
scraper = AvitoScraper()
|
|
mock_browser = _make_mock_browser(_NORMAL_HTML)
|
|
scraper._browser = mock_browser
|
|
|
|
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
|
|
|
|
assert result == _NORMAL_HTML
|
|
mock_browser.fetch.assert_awaited_once_with("https://www.avito.ru/test")
|
|
# _cffi was never set — assert remains None
|
|
assert scraper._cffi is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 2. curl branch unchanged
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_serp_html_curl_mode_uses_cffi() -> None:
|
|
"""Default mode (_browser is None) → _fetch_serp_html goes through curl path."""
|
|
scraper = AvitoScraper()
|
|
|
|
# Mock the curl session returning HTTP 200 with normal HTML
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 200
|
|
mock_response.text = _NORMAL_HTML
|
|
|
|
mock_cffi = MagicMock()
|
|
mock_cffi.get = AsyncMock(return_value=mock_response)
|
|
scraper._cffi = mock_cffi
|
|
|
|
# _browser must be None (default curl path)
|
|
assert scraper._browser is None
|
|
|
|
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
|
|
|
|
assert result == _NORMAL_HTML
|
|
mock_cffi.get.assert_awaited_once_with("https://www.avito.ru/test")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 3. firewall-detect in browser-mode → AvitoBlockedError (no rotate URL)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_browser_mode_firewall_raises_blocked_error() -> None:
|
|
"""Browser firewall + no rotate URL + no curl fallback (_cffi=None) → AvitoBlockedError."""
|
|
scraper = AvitoScraper()
|
|
mock_browser = _make_mock_browser(_FIREWALL_HTML)
|
|
scraper._browser = mock_browser
|
|
scraper._cffi = None # no curl_cffi fallback available (pipeline shared-browser path)
|
|
|
|
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
|
mock_settings.avito_proxy_rotate_url = None # no rotation configured
|
|
mock_settings.avito_proxy_max_rotations = 2
|
|
|
|
with pytest.raises(AvitoBlockedError, match="browser-mode"):
|
|
await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 3b. #1805 — curl_cffi fallback when browser returns firewall
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_browser_firewall_falls_back_to_cffi_returns_html() -> None:
|
|
"""#1805: browser firewall + curl_cffi fallback armed → curl returns clean HTML."""
|
|
scraper = AvitoScraper()
|
|
scraper._browser = _make_mock_browser(_FIREWALL_HTML)
|
|
|
|
# curl_cffi fallback: HTTP 200 with clean (non-firewall) HTML.
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.text = _NORMAL_HTML
|
|
mock_cffi = MagicMock()
|
|
mock_cffi.get = AsyncMock(return_value=mock_resp)
|
|
scraper._cffi = mock_cffi
|
|
|
|
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
|
mock_settings.avito_proxy_rotate_url = None
|
|
mock_settings.avito_proxy_max_rotations = 0
|
|
mock_settings.scraper_proxy_url = None
|
|
|
|
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
|
|
|
|
assert result == _NORMAL_HTML
|
|
mock_cffi.get.assert_awaited_once_with("https://www.avito.ru/test")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_browser_firewall_cffi_fallback_also_firewall_raises() -> None:
|
|
"""#1805: browser firewall AND curl_cffi fallback also firewall → AvitoBlockedError."""
|
|
scraper = AvitoScraper()
|
|
scraper._browser = _make_mock_browser(_FIREWALL_HTML)
|
|
|
|
# curl_cffi fallback: HTTP 200 but ALSO firewall HTML → no escape.
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.text = _FIREWALL_HTML
|
|
mock_cffi = MagicMock()
|
|
mock_cffi.get = AsyncMock(return_value=mock_resp)
|
|
scraper._cffi = mock_cffi
|
|
|
|
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
|
mock_settings.avito_proxy_rotate_url = None
|
|
mock_settings.avito_proxy_max_rotations = 0
|
|
mock_settings.scraper_proxy_url = None # no backconnect reconnect retries
|
|
|
|
# curl fallback raises its own AvitoBlockedError (firewall на HTTP 200);
|
|
# browser branch re-raises it as a hard-block.
|
|
with pytest.raises(AvitoBlockedError):
|
|
await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_browser_mode_firewall_rotates_and_retries() -> None:
|
|
"""Browser returns firewall on first call, normal HTML after rotation → returns HTML."""
|
|
scraper = AvitoScraper()
|
|
|
|
# First call → firewall; second call → normal HTML
|
|
mock_browser = MagicMock()
|
|
mock_browser.fetch = AsyncMock(side_effect=[_FIREWALL_HTML, _NORMAL_HTML])
|
|
scraper._browser = mock_browser
|
|
|
|
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
|
mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/rotate"
|
|
mock_settings.avito_proxy_max_rotations = 1
|
|
|
|
with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=True)):
|
|
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=2)
|
|
|
|
assert result == _NORMAL_HTML
|
|
assert mock_browser.fetch.await_count == 2
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 4. pagination terminates via empty-parse
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_city_wide_browser_mode_stops_on_empty_parse() -> None:
|
|
"""fetch_city_wide with browser mode: page=1 returns lots, page=2 returns [] → stops.
|
|
|
|
Verifies the documented invariant: callers break on `not lots` regardless of
|
|
fetch mode. _fetch_serp_html is mocked at method level (browser vs curl is
|
|
already covered by unit tests above); here we confirm the caller loop behaviour.
|
|
"""
|
|
scraper = AvitoScraper()
|
|
|
|
html_p1 = "<html>page1</html>"
|
|
html_p2 = "<html>page2</html>"
|
|
|
|
lot = ScrapedLot(
|
|
source="avito",
|
|
source_url="https://www.avito.ru/ekaterinburg/kvartiry/test",
|
|
source_id="LOT_1",
|
|
price_rub=5_000_000,
|
|
)
|
|
|
|
call_count = 0
|
|
|
|
async def mock_fetch_html(url: str, page: int) -> str:
|
|
return html_p1 if page == 1 else html_p2
|
|
|
|
def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
|
nonlocal call_count
|
|
call_count += 1
|
|
return [lot] if "page1" in html else []
|
|
|
|
with patch.object(scraper, "_fetch_serp_html", side_effect=mock_fetch_html):
|
|
with patch.object(scraper, "_parse_html", side_effect=mock_parse):
|
|
with patch.object(scraper, "sleep_between_requests", new=AsyncMock()):
|
|
result = await scraper.fetch_city_wide(pages=10, delay_override_sec=0)
|
|
|
|
assert len(result) == 1
|
|
assert result[0].source_id == "LOT_1"
|
|
# page1 parsed (returned lot) + page2 parsed (returned []) → stopped; page3-10 never called
|
|
assert call_count == 2
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 5. rotation returns False → immediate AvitoBlockedError
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_browser_mode_rotation_returns_false_raises_blocked() -> None:
|
|
"""rotate_url set but _rotate_ip() returns False → must immediately raise AvitoBlockedError.
|
|
|
|
Covers the branch: firewall detected + rotate_url configured + _rotate_ip() == False
|
|
(e.g. rotation API timeout/failure) → we cannot recover → blocked.
|
|
"""
|
|
scraper = AvitoScraper()
|
|
scraper._browser = _make_mock_browser(_FIREWALL_HTML)
|
|
|
|
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
|
mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
|
|
mock_settings.avito_proxy_max_rotations = 2
|
|
with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=False)):
|
|
with pytest.raises(AvitoBlockedError, match="browser-mode"):
|
|
await scraper._fetch_serp_html("https://avito.ru/test", page=1)
|