Avito

"""Tests for BrowserFetcher (#884, Phase 0). Полностью замоканы: не требуют установленного Firefox или camoufox. Покрывают: - _proxy_dict(): парсинг proxy URL с credentials и без - fetch(): new_page → goto → wait_for_timeout → content → close - recycle: счётчик страниц → relaunch браузера при достижении порога - crash-recovery: TargetClosedError → relaunch → retry """ from __future__ import annotations import os from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch import pytest os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") # ── helpers ──────────────────────────────────────────────────────────────────── def _mock_settings( proxy_url: str | None = None, recycle_pages: int = 15, nav_timeout_ms: int = 60000, wait_ms: int = 2500, ) -> SimpleNamespace: return SimpleNamespace( scraper_proxy_url=proxy_url, browser_recycle_pages=recycle_pages, browser_nav_timeout_ms=nav_timeout_ms, browser_wait_ms=wait_ms, ) def _make_mock_page(html: str = "ok") -> MagicMock: """Создаёт мок страницы playwright с нужными async-методами.""" page = MagicMock() page.goto = AsyncMock() page.wait_for_timeout = AsyncMock() page.content = AsyncMock(return_value=html) page.close = AsyncMock() return page def _make_mock_browser(page: MagicMock | None = None) -> MagicMock: """Создаёт мок browser-контекста camoufox.""" if page is None: page = _make_mock_page() browser = MagicMock() browser.new_page = AsyncMock(return_value=page) return browser def _make_mock_camoufox_ctx(browser: MagicMock) -> MagicMock: """Мок AsyncCamoufox(...) context manager → возвращает browser из __aenter__.""" ctx = MagicMock() ctx.__aenter__ = AsyncMock(return_value=browser) ctx.__aexit__ = AsyncMock(return_value=None) return ctx # ── _proxy_dict() ────────────────────────────────────────────────────────────── # _proxy_dict() импортирует settings через `from app.core.config import settings` # внутри метода, поэтому патчим app.core.config.settings (источник). def test_proxy_dict_parses_full_url(): """_proxy_dict() разбирает URL с username/password → playwright dict.""" from app.services.scrapers.browser_fetcher import BrowserFetcher fetcher = BrowserFetcher() ms = _mock_settings(proxy_url="http://user123:pass456@proxy.example.com:8080") with patch("app.core.config.settings", ms): result = fetcher._proxy_dict() assert result == { "server": "http://proxy.example.com:8080", "username": "user123", "password": "pass456", } def test_proxy_dict_no_credentials(): """_proxy_dict() для URL без credentials возвращает только server.""" from app.services.scrapers.browser_fetcher import BrowserFetcher fetcher = BrowserFetcher() ms = _mock_settings(proxy_url="http://proxy.example.com:3128") with patch("app.core.config.settings", ms): result = fetcher._proxy_dict() assert result == {"server": "http://proxy.example.com:3128"} assert "username" not in result assert "password" not in result def test_proxy_dict_returns_none_when_no_proxy(): """_proxy_dict() → None когда scraper_proxy_url не задан.""" from app.services.scrapers.browser_fetcher import BrowserFetcher fetcher = BrowserFetcher() ms = _mock_settings(proxy_url=None) with patch("app.core.config.settings", ms): result = fetcher._proxy_dict() assert result is None # ── fetch(): new_page → goto → content → close ──────────────────────────────── @pytest.mark.asyncio async def test_fetch_calls_page_lifecycle(): """fetch() вызывает new_page, goto, wait_for_timeout, content, close.""" from app.services.scrapers.browser_fetcher import BrowserFetcher expected_html = "test" page = _make_mock_page(html=expected_html) browser = _make_mock_browser(page=page) ctx = _make_mock_camoufox_ctx(browser) ms = _mock_settings(recycle_pages=15, nav_timeout_ms=60000, wait_ms=2500) with patch("app.core.config.settings", ms): with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx): fetcher = BrowserFetcher() async with fetcher: result = await fetcher.fetch("https://example.com") assert result == expected_html browser.new_page.assert_called_once() page.goto.assert_called_once_with( "https://example.com", timeout=60000, wait_until="domcontentloaded" ) page.wait_for_timeout.assert_called_once_with(2500) page.content.assert_called_once() page.close.assert_called_once() @pytest.mark.asyncio async def test_fetch_returns_html_string(): """fetch() возвращает строку HTML от page.content().""" from app.services.scrapers.browser_fetcher import BrowserFetcher html = "Avito" page = _make_mock_page(html=html) browser = _make_mock_browser(page=page) ctx = _make_mock_camoufox_ctx(browser) ms = _mock_settings() with patch("app.core.config.settings", ms): with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx): fetcher = BrowserFetcher() async with fetcher: result = await fetcher.fetch("https://avito.ru/test") assert result == html assert isinstance(result, str) # ── recycle: relaunch at threshold ──────────────────────────────────────────── @pytest.mark.asyncio async def test_recycle_triggers_relaunch_at_threshold(): """После recycle_pages страниц браузер перезапускается (relaunch).""" from app.services.scrapers.browser_fetcher import BrowserFetcher recycle_after = 3 page = _make_mock_page() browser = _make_mock_browser(page=page) ctx = _make_mock_camoufox_ctx(browser) ms = _mock_settings(recycle_pages=recycle_after, wait_ms=0) launch_count = 0 original_launch = BrowserFetcher._launch async def counting_launch(self: BrowserFetcher) -> None: nonlocal launch_count launch_count += 1 await original_launch(self) with patch("app.core.config.settings", ms): with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx): with patch.object(BrowserFetcher, "_launch", counting_launch): fetcher = BrowserFetcher() async with fetcher: # Первый запуск через __aenter__ assert launch_count == 1 # Отправляем recycle_after страниц — на последней сработает relaunch for _ in range(recycle_after): await fetcher.fetch("https://example.com") # После recycle браузер должен был перезапуститься ещё раз assert launch_count == 2 @pytest.mark.asyncio async def test_page_counter_resets_after_relaunch(): """После recycle page_count сбрасывается в 0.""" from app.services.scrapers.browser_fetcher import BrowserFetcher recycle_after = 2 page = _make_mock_page() browser = _make_mock_browser(page=page) ctx = _make_mock_camoufox_ctx(browser) ms = _mock_settings(recycle_pages=recycle_after, wait_ms=0) with patch("app.core.config.settings", ms): with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx): fetcher = BrowserFetcher() async with fetcher: for _ in range(recycle_after): await fetcher.fetch("https://example.com") # После recycle счётчик сброшен assert fetcher._page_count == 0 # ── crash-recovery ───────────────────────────────────────────────────────────── class _FakeTargetClosedError(Exception): """Имитирует playwright TargetClosedError (проверка по имени класса).""" _FakeTargetClosedError.__name__ = "TargetClosedError" _FakeTargetClosedError.__qualname__ = "TargetClosedError" @pytest.mark.asyncio async def test_fetch_retries_once_on_target_closed_error(): """fetch() при TargetClosedError перезапускает браузер и повторяет запрос.""" from app.services.scrapers.browser_fetcher import BrowserFetcher html = "retry-ok" # Первая страница бросает краш, вторая отвечает нормально crash_page = MagicMock() crash_page.goto = AsyncMock(side_effect=_FakeTargetClosedError("target closed")) crash_page.close = AsyncMock() ok_page = _make_mock_page(html=html) browser = _make_mock_browser() call_num = 0 async def new_page_side_effect() -> MagicMock: nonlocal call_num call_num += 1 return crash_page if call_num == 1 else ok_page browser.new_page = AsyncMock(side_effect=new_page_side_effect) ctx = _make_mock_camoufox_ctx(browser) ms = _mock_settings(wait_ms=0) with patch("app.core.config.settings", ms): with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx): fetcher = BrowserFetcher() async with fetcher: result = await fetcher.fetch("https://avito.ru/page") assert result == html # 2 обращения к new_page: первое (crash) + второе (retry после relaunch) assert call_num == 2 @pytest.mark.asyncio async def test_fetch_propagates_non_crash_exception(): """fetch() не подавляет обычные (не-crash) исключения.""" from app.services.scrapers.browser_fetcher import BrowserFetcher page = MagicMock() page.goto = AsyncMock(side_effect=ValueError("unexpected parse error")) page.close = AsyncMock() browser = _make_mock_browser(page=page) ctx = _make_mock_camoufox_ctx(browser) ms = _mock_settings(wait_ms=0) with patch("app.core.config.settings", ms): with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx): fetcher = BrowserFetcher() async with fetcher: with pytest.raises(ValueError, match="unexpected parse error"): await fetcher.fetch("https://avito.ru/bad")