Avito

"""Tests for BrowserFetcher (#884/#905, connect-mode). Полностью замоканы: не требуют установленного Firefox, camoufox или реального WS-сервера. Покрывают: - fetch(): new_page → goto → wait_for_timeout → content → close - recycle: счётчик страниц → reconnect при достижении порога - crash-recovery: TargetClosedError → reconnect → retry - page counter сбрасывается после reconnect - non-crash exception пробрасывается без перехвата """ from __future__ import annotations import os from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch import pytest os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") # ── helpers ──────────────────────────────────────────────────────────────────── def _mock_settings( proxy_url: str | None = None, recycle_pages: int = 15, nav_timeout_ms: int = 60000, wait_ms: int = 2500, ws_endpoint: str = "ws://fake-browser:3000/playwright", ) -> SimpleNamespace: return SimpleNamespace( scraper_proxy_url=proxy_url, browser_recycle_pages=recycle_pages, browser_nav_timeout_ms=nav_timeout_ms, browser_wait_ms=wait_ms, browser_ws_endpoint=ws_endpoint, ) def _make_mock_page(html: str = "ok") -> MagicMock: """Создаёт мок страницы playwright с нужными async-методами.""" page = MagicMock() page.goto = AsyncMock() page.wait_for_timeout = AsyncMock() page.content = AsyncMock(return_value=html) page.close = AsyncMock() return page def _make_playwright_stack( page: MagicMock | None = None, html: str = "ok", ) -> tuple[MagicMock, MagicMock, MagicMock]: """Создаёт полную mock-цепочку playwright: pw_cm → pw → browser → page. Returns: (pw_cm, pw_mock, browser_mock) """ if page is None: page = _make_mock_page(html=html) browser_mock = MagicMock() browser_mock.new_page = AsyncMock(return_value=page) browser_mock.close = AsyncMock() firefox_mock = MagicMock() firefox_mock.connect = AsyncMock(return_value=browser_mock) pw_mock = MagicMock() pw_mock.firefox = firefox_mock pw_mock.stop = AsyncMock() async def _start() -> MagicMock: return pw_mock pw_cm = MagicMock() pw_cm.start = _start return pw_cm, pw_mock, browser_mock # ── fetch(): new_page → goto → content → close ──────────────────────────────── @pytest.mark.asyncio async def test_fetch_calls_page_lifecycle(): """fetch() вызывает new_page, goto, wait_for_timeout, content, close.""" from app.services.scrapers.browser_fetcher import BrowserFetcher expected_html = "test" page = _make_mock_page(html=expected_html) pw_cm, _pw, browser = _make_playwright_stack(page=page) ms = _mock_settings(recycle_pages=15, nav_timeout_ms=60000, wait_ms=2500) with patch("app.core.config.settings", ms): with patch("playwright.async_api.async_playwright", return_value=pw_cm): fetcher = BrowserFetcher() async with fetcher: result = await fetcher.fetch("https://example.com") assert result == expected_html browser.new_page.assert_called_once() page.goto.assert_called_once_with( "https://example.com", timeout=60000, wait_until="domcontentloaded" ) page.wait_for_timeout.assert_called_once_with(2500) page.content.assert_called_once() page.close.assert_called_once() @pytest.mark.asyncio async def test_fetch_returns_html_string(): """fetch() возвращает строку HTML от page.content().""" from app.services.scrapers.browser_fetcher import BrowserFetcher html = "Avito" pw_cm, _pw, _browser = _make_playwright_stack(html=html) ms = _mock_settings() with patch("app.core.config.settings", ms): with patch("playwright.async_api.async_playwright", return_value=pw_cm): fetcher = BrowserFetcher() async with fetcher: result = await fetcher.fetch("https://avito.ru/test") assert result == html assert isinstance(result, str) # ── recycle: reconnect at threshold ─────────────────────────────────────────── @pytest.mark.asyncio async def test_recycle_triggers_reconnect_at_threshold(): """После recycle_pages страниц выполняется переподключение (_reconnect).""" from app.services.scrapers.browser_fetcher import BrowserFetcher recycle_after = 3 pw_cm, _pw, _browser = _make_playwright_stack() ms = _mock_settings(recycle_pages=recycle_after, wait_ms=0) reconnect_count = 0 original_reconnect = BrowserFetcher._reconnect async def counting_reconnect(self: BrowserFetcher) -> None: nonlocal reconnect_count reconnect_count += 1 await original_reconnect(self) with patch("app.core.config.settings", ms): with patch("playwright.async_api.async_playwright", return_value=pw_cm): with patch.object(BrowserFetcher, "_reconnect", counting_reconnect): fetcher = BrowserFetcher() async with fetcher: assert reconnect_count == 0 for _ in range(recycle_after): await fetcher.fetch("https://example.com") # Третья страница триггерит recycle assert reconnect_count == 1 @pytest.mark.asyncio async def test_page_counter_resets_after_reconnect(): """После recycle _page_count сбрасывается в 0.""" from app.services.scrapers.browser_fetcher import BrowserFetcher recycle_after = 2 pw_cm, _pw, _browser = _make_playwright_stack() ms = _mock_settings(recycle_pages=recycle_after, wait_ms=0) with patch("app.core.config.settings", ms): with patch("playwright.async_api.async_playwright", return_value=pw_cm): fetcher = BrowserFetcher() async with fetcher: for _ in range(recycle_after): await fetcher.fetch("https://example.com") # После recycle счётчик сброшен assert fetcher._page_count == 0 # ── crash-recovery ───────────────────────────────────────────────────────────── class _FakeTargetClosedError(Exception): """Имитирует playwright TargetClosedError (проверка по имени класса).""" _FakeTargetClosedError.__name__ = "TargetClosedError" _FakeTargetClosedError.__qualname__ = "TargetClosedError" @pytest.mark.asyncio async def test_fetch_retries_once_on_target_closed_error(): """fetch() при TargetClosedError переподключается и повторяет запрос.""" from app.services.scrapers.browser_fetcher import BrowserFetcher html = "retry-ok" # Первая страница кидает краш, вторая отвечает нормально crash_page = MagicMock() crash_page.goto = AsyncMock(side_effect=_FakeTargetClosedError("target closed")) crash_page.close = AsyncMock() ok_page = _make_mock_page(html=html) browser_mock = MagicMock() browser_mock.close = AsyncMock() call_num = 0 async def new_page_side_effect() -> MagicMock: nonlocal call_num call_num += 1 return crash_page if call_num == 1 else ok_page browser_mock.new_page = AsyncMock(side_effect=new_page_side_effect) firefox_mock = MagicMock() firefox_mock.connect = AsyncMock(return_value=browser_mock) pw_mock = MagicMock() pw_mock.firefox = firefox_mock pw_mock.stop = AsyncMock() async def _start() -> MagicMock: return pw_mock pw_cm = MagicMock() pw_cm.start = _start ms = _mock_settings(wait_ms=0) with patch("app.core.config.settings", ms): with patch("playwright.async_api.async_playwright", return_value=pw_cm): fetcher = BrowserFetcher() async with fetcher: result = await fetcher.fetch("https://avito.ru/page") assert result == html assert call_num == 2 # connect вызывался дважды: при __aenter__ + при reconnect assert firefox_mock.connect.await_count == 2 @pytest.mark.asyncio async def test_fetch_propagates_non_crash_exception(): """fetch() не подавляет обычные (не-crash) исключения.""" from app.services.scrapers.browser_fetcher import BrowserFetcher page = MagicMock() page.goto = AsyncMock(side_effect=ValueError("unexpected parse error")) page.close = AsyncMock() pw_cm, _pw, _browser = _make_playwright_stack(page=page) ms = _mock_settings(wait_ms=0) with patch("app.core.config.settings", ms): with patch("playwright.async_api.async_playwright", return_value=pw_cm): fetcher = BrowserFetcher() async with fetcher: with pytest.raises(ValueError, match="unexpected parse error"): await fetcher.fetch("https://avito.ru/bad")