bot-backend
441e48141b
All checks were successful
Deploy Trade-In / changes (push) Successful in 4s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 31s
Deploy Trade-In / build-backend (push) Successful in 44s
Deploy Trade-In / deploy (push) Successful in 36s
Co-authored-by: bot-backend <bot-backend@gendsgn.local> Co-committed-by: bot-backend <bot-backend@gendsgn.local>
225 lines
8.7 KiB
Python
225 lines
8.7 KiB
Python
"""#901 — avito SERP fetch via BrowserFetcher behind scraper_fetch_mode flag.
|
|
|
|
Tests:
|
|
1. browser branch chosen: mode="browser" → _browser set, _cffi is None, HTML from mock browser.
|
|
2. curl branch unchanged: default mode → _browser is None, _cffi session used.
|
|
3. firewall-detect in browser-mode: firewall HTML + no rotate URL → AvitoBlockedError.
|
|
4. pagination terminates via empty-parse: browser HTML with 0 lots → caller loop breaks.
|
|
|
|
Без сети, без БД, без curl_cffi.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
|
|
|
from app.services.scrapers.avito import _FIREWALL_MARKERS, AvitoScraper
|
|
from app.services.scrapers.avito_exceptions import AvitoBlockedError
|
|
from app.services.scrapers.base import ScrapedLot
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_NORMAL_HTML = "<html><body>normal avito SERP</body></html>"
|
|
_FIREWALL_HTML = f"<html><head><title>{_FIREWALL_MARKERS[0]}</title></head></html>"
|
|
|
|
|
|
def _make_mock_browser(html: str = _NORMAL_HTML) -> MagicMock:
|
|
"""Return a mock BrowserFetcher context manager that yields html from fetch()."""
|
|
mock = MagicMock()
|
|
mock.__aenter__ = AsyncMock(return_value=mock)
|
|
mock.__aexit__ = AsyncMock(return_value=None)
|
|
mock.fetch = AsyncMock(return_value=html)
|
|
return mock
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 1. browser branch chosen
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_aenter_browser_mode_sets_browser_attribute() -> None:
|
|
"""mode=browser → __aenter__ instantiates BrowserFetcher, _cffi stays None."""
|
|
mock_browser = _make_mock_browser()
|
|
scraper = AvitoScraper()
|
|
|
|
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
|
mock_settings.scraper_fetch_mode = "browser"
|
|
mock_settings.scraper_proxy_url = None
|
|
mock_settings.avito_proxy_rotate_url = None
|
|
mock_settings.avito_proxy_max_rotations = 2
|
|
|
|
with patch(
|
|
"app.services.scrapers.avito.BrowserFetcher",
|
|
return_value=mock_browser,
|
|
):
|
|
# Patch BaseScraper.__aenter__ so we skip its real setup (DB/network)
|
|
with patch(
|
|
"app.services.scrapers.base.BaseScraper.__aenter__",
|
|
new=AsyncMock(return_value=scraper),
|
|
):
|
|
result = await scraper.__aenter__()
|
|
|
|
assert result is scraper
|
|
assert scraper._browser is mock_browser
|
|
assert scraper._cffi is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_serp_html_browser_mode_returns_html() -> None:
|
|
"""_fetch_serp_html with _browser set returns HTML string, never touches _cffi."""
|
|
scraper = AvitoScraper()
|
|
mock_browser = _make_mock_browser(_NORMAL_HTML)
|
|
scraper._browser = mock_browser
|
|
|
|
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
|
|
|
|
assert result == _NORMAL_HTML
|
|
mock_browser.fetch.assert_awaited_once_with("https://www.avito.ru/test")
|
|
# _cffi was never set — assert remains None
|
|
assert scraper._cffi is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 2. curl branch unchanged
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_serp_html_curl_mode_uses_cffi() -> None:
|
|
"""Default mode (_browser is None) → _fetch_serp_html goes through curl path."""
|
|
scraper = AvitoScraper()
|
|
|
|
# Mock the curl session returning HTTP 200 with normal HTML
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 200
|
|
mock_response.text = _NORMAL_HTML
|
|
|
|
mock_cffi = MagicMock()
|
|
mock_cffi.get = AsyncMock(return_value=mock_response)
|
|
scraper._cffi = mock_cffi
|
|
|
|
# _browser must be None (default curl path)
|
|
assert scraper._browser is None
|
|
|
|
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
|
|
|
|
assert result == _NORMAL_HTML
|
|
mock_cffi.get.assert_awaited_once_with("https://www.avito.ru/test")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 3. firewall-detect in browser-mode → AvitoBlockedError (no rotate URL)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_browser_mode_firewall_raises_blocked_error() -> None:
|
|
"""Browser returns firewall page + no rotate URL configured → AvitoBlockedError."""
|
|
scraper = AvitoScraper()
|
|
mock_browser = _make_mock_browser(_FIREWALL_HTML)
|
|
scraper._browser = mock_browser
|
|
|
|
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
|
mock_settings.avito_proxy_rotate_url = None # no rotation configured
|
|
mock_settings.avito_proxy_max_rotations = 2
|
|
|
|
with pytest.raises(AvitoBlockedError, match="browser-mode"):
|
|
await scraper._fetch_serp_html("https://www.avito.ru/test", page=1)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_browser_mode_firewall_rotates_and_retries() -> None:
|
|
"""Browser returns firewall on first call, normal HTML after rotation → returns HTML."""
|
|
scraper = AvitoScraper()
|
|
|
|
# First call → firewall; second call → normal HTML
|
|
mock_browser = MagicMock()
|
|
mock_browser.fetch = AsyncMock(side_effect=[_FIREWALL_HTML, _NORMAL_HTML])
|
|
scraper._browser = mock_browser
|
|
|
|
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
|
mock_settings.avito_proxy_rotate_url = "http://proxy.example.com/rotate"
|
|
mock_settings.avito_proxy_max_rotations = 1
|
|
|
|
with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=True)):
|
|
result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=2)
|
|
|
|
assert result == _NORMAL_HTML
|
|
assert mock_browser.fetch.await_count == 2
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 4. pagination terminates via empty-parse
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_city_wide_browser_mode_stops_on_empty_parse() -> None:
|
|
"""fetch_city_wide with browser mode: page=1 returns lots, page=2 returns [] → stops.
|
|
|
|
Verifies the documented invariant: callers break on `not lots` regardless of
|
|
fetch mode. _fetch_serp_html is mocked at method level (browser vs curl is
|
|
already covered by unit tests above); here we confirm the caller loop behaviour.
|
|
"""
|
|
scraper = AvitoScraper()
|
|
|
|
html_p1 = "<html>page1</html>"
|
|
html_p2 = "<html>page2</html>"
|
|
|
|
lot = ScrapedLot(
|
|
source="avito",
|
|
source_url="https://www.avito.ru/ekaterinburg/kvartiry/test",
|
|
source_id="LOT_1",
|
|
price_rub=5_000_000,
|
|
)
|
|
|
|
call_count = 0
|
|
|
|
async def mock_fetch_html(url: str, page: int) -> str:
|
|
return html_p1 if page == 1 else html_p2
|
|
|
|
def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
|
nonlocal call_count
|
|
call_count += 1
|
|
return [lot] if "page1" in html else []
|
|
|
|
with patch.object(scraper, "_fetch_serp_html", side_effect=mock_fetch_html):
|
|
with patch.object(scraper, "_parse_html", side_effect=mock_parse):
|
|
with patch.object(scraper, "sleep_between_requests", new=AsyncMock()):
|
|
result = await scraper.fetch_city_wide(pages=10, delay_override_sec=0)
|
|
|
|
assert len(result) == 1
|
|
assert result[0].source_id == "LOT_1"
|
|
# page1 parsed (returned lot) + page2 parsed (returned []) → stopped; page3-10 never called
|
|
assert call_count == 2
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 5. rotation returns False → immediate AvitoBlockedError
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_browser_mode_rotation_returns_false_raises_blocked() -> None:
|
|
"""rotate_url set but _rotate_ip() returns False → must immediately raise AvitoBlockedError.
|
|
|
|
Covers the branch: firewall detected + rotate_url configured + _rotate_ip() == False
|
|
(e.g. rotation API timeout/failure) → we cannot recover → blocked.
|
|
"""
|
|
scraper = AvitoScraper()
|
|
scraper._browser = _make_mock_browser(_FIREWALL_HTML)
|
|
|
|
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
|
mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
|
|
mock_settings.avito_proxy_max_rotations = 2
|
|
with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=False)):
|
|
with pytest.raises(AvitoBlockedError, match="browser-mode"):
|
|
await scraper._fetch_serp_html("https://avito.ru/test", page=1)
|