bot-backend
e03f4e4534
Epic #883 foundation. Provider-agnostic BrowserFetcher (camoufox headless Firefox): async ctx mgr + page-recycle + crash-recovery + proxy. camoufox/playwright lazy imports (won't break app import). scraper_fetch_mode defaults curl_cffi -> DORMANT (no parser/estimator/avito/cian change). config: 4 browser_* settings. Dockerfile: +5 Firefox libs + camoufox fetch layer. compose: mem_limit 2.5g. (uv.lock gitignored -> re-resolved at build.) 9 mocked tests pass, ruff clean. WARN: Docker build + container fetch UNVERIFIED (no local Docker); ~960MB image; merge auto-deploys to prod. MERGE-GATE post-demo + CI-build-verify. Refs #884, #883
295 lines
11 KiB
Python
295 lines
11 KiB
Python
"""Tests for BrowserFetcher (#884, Phase 0).
|
||
|
||
Полностью замоканы: не требуют установленного Firefox или camoufox.
|
||
Покрывают:
|
||
- _proxy_dict(): парсинг proxy URL с credentials и без
|
||
- fetch(): new_page → goto → wait_for_timeout → content → close
|
||
- recycle: счётчик страниц → relaunch браузера при достижении порога
|
||
- crash-recovery: TargetClosedError → relaunch → retry
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from types import SimpleNamespace
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
|
||
# ── helpers ────────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _mock_settings(
|
||
proxy_url: str | None = None,
|
||
recycle_pages: int = 15,
|
||
nav_timeout_ms: int = 60000,
|
||
wait_ms: int = 2500,
|
||
) -> SimpleNamespace:
|
||
return SimpleNamespace(
|
||
scraper_proxy_url=proxy_url,
|
||
browser_recycle_pages=recycle_pages,
|
||
browser_nav_timeout_ms=nav_timeout_ms,
|
||
browser_wait_ms=wait_ms,
|
||
)
|
||
|
||
|
||
def _make_mock_page(html: str = "<html>ok</html>") -> MagicMock:
|
||
"""Создаёт мок страницы playwright с нужными async-методами."""
|
||
page = MagicMock()
|
||
page.goto = AsyncMock()
|
||
page.wait_for_timeout = AsyncMock()
|
||
page.content = AsyncMock(return_value=html)
|
||
page.close = AsyncMock()
|
||
return page
|
||
|
||
|
||
def _make_mock_browser(page: MagicMock | None = None) -> MagicMock:
|
||
"""Создаёт мок browser-контекста camoufox."""
|
||
if page is None:
|
||
page = _make_mock_page()
|
||
browser = MagicMock()
|
||
browser.new_page = AsyncMock(return_value=page)
|
||
return browser
|
||
|
||
|
||
def _make_mock_camoufox_ctx(browser: MagicMock) -> MagicMock:
|
||
"""Мок AsyncCamoufox(...) context manager → возвращает browser из __aenter__."""
|
||
ctx = MagicMock()
|
||
ctx.__aenter__ = AsyncMock(return_value=browser)
|
||
ctx.__aexit__ = AsyncMock(return_value=None)
|
||
return ctx
|
||
|
||
|
||
# ── _proxy_dict() ──────────────────────────────────────────────────────────────
|
||
# _proxy_dict() импортирует settings через `from app.core.config import settings`
|
||
# внутри метода, поэтому патчим app.core.config.settings (источник).
|
||
|
||
|
||
def test_proxy_dict_parses_full_url():
|
||
"""_proxy_dict() разбирает URL с username/password → playwright dict."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
fetcher = BrowserFetcher()
|
||
ms = _mock_settings(proxy_url="http://user123:pass456@proxy.example.com:8080")
|
||
with patch("app.core.config.settings", ms):
|
||
result = fetcher._proxy_dict()
|
||
|
||
assert result == {
|
||
"server": "http://proxy.example.com:8080",
|
||
"username": "user123",
|
||
"password": "pass456",
|
||
}
|
||
|
||
|
||
def test_proxy_dict_no_credentials():
|
||
"""_proxy_dict() для URL без credentials возвращает только server."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
fetcher = BrowserFetcher()
|
||
ms = _mock_settings(proxy_url="http://proxy.example.com:3128")
|
||
with patch("app.core.config.settings", ms):
|
||
result = fetcher._proxy_dict()
|
||
|
||
assert result == {"server": "http://proxy.example.com:3128"}
|
||
assert "username" not in result
|
||
assert "password" not in result
|
||
|
||
|
||
def test_proxy_dict_returns_none_when_no_proxy():
|
||
"""_proxy_dict() → None когда scraper_proxy_url не задан."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
fetcher = BrowserFetcher()
|
||
ms = _mock_settings(proxy_url=None)
|
||
with patch("app.core.config.settings", ms):
|
||
result = fetcher._proxy_dict()
|
||
|
||
assert result is None
|
||
|
||
|
||
# ── fetch(): new_page → goto → content → close ────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_calls_page_lifecycle():
|
||
"""fetch() вызывает new_page, goto, wait_for_timeout, content, close."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
expected_html = "<html><body>test</body></html>"
|
||
page = _make_mock_page(html=expected_html)
|
||
browser = _make_mock_browser(page=page)
|
||
ctx = _make_mock_camoufox_ctx(browser)
|
||
|
||
ms = _mock_settings(recycle_pages=15, nav_timeout_ms=60000, wait_ms=2500)
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
result = await fetcher.fetch("https://example.com")
|
||
|
||
assert result == expected_html
|
||
browser.new_page.assert_called_once()
|
||
page.goto.assert_called_once_with(
|
||
"https://example.com", timeout=60000, wait_until="domcontentloaded"
|
||
)
|
||
page.wait_for_timeout.assert_called_once_with(2500)
|
||
page.content.assert_called_once()
|
||
page.close.assert_called_once()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_returns_html_string():
|
||
"""fetch() возвращает строку HTML от page.content()."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
html = "<html><title>Avito</title></html>"
|
||
page = _make_mock_page(html=html)
|
||
browser = _make_mock_browser(page=page)
|
||
ctx = _make_mock_camoufox_ctx(browser)
|
||
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
result = await fetcher.fetch("https://avito.ru/test")
|
||
|
||
assert result == html
|
||
assert isinstance(result, str)
|
||
|
||
|
||
# ── recycle: relaunch at threshold ────────────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_recycle_triggers_relaunch_at_threshold():
|
||
"""После recycle_pages страниц браузер перезапускается (relaunch)."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
recycle_after = 3
|
||
|
||
page = _make_mock_page()
|
||
browser = _make_mock_browser(page=page)
|
||
ctx = _make_mock_camoufox_ctx(browser)
|
||
|
||
ms = _mock_settings(recycle_pages=recycle_after, wait_ms=0)
|
||
|
||
launch_count = 0
|
||
original_launch = BrowserFetcher._launch
|
||
|
||
async def counting_launch(self: BrowserFetcher) -> None:
|
||
nonlocal launch_count
|
||
launch_count += 1
|
||
await original_launch(self)
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx):
|
||
with patch.object(BrowserFetcher, "_launch", counting_launch):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
# Первый запуск через __aenter__
|
||
assert launch_count == 1
|
||
# Отправляем recycle_after страниц — на последней сработает relaunch
|
||
for _ in range(recycle_after):
|
||
await fetcher.fetch("https://example.com")
|
||
# После recycle браузер должен был перезапуститься ещё раз
|
||
assert launch_count == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_page_counter_resets_after_relaunch():
|
||
"""После recycle page_count сбрасывается в 0."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
recycle_after = 2
|
||
|
||
page = _make_mock_page()
|
||
browser = _make_mock_browser(page=page)
|
||
ctx = _make_mock_camoufox_ctx(browser)
|
||
|
||
ms = _mock_settings(recycle_pages=recycle_after, wait_ms=0)
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
for _ in range(recycle_after):
|
||
await fetcher.fetch("https://example.com")
|
||
# После recycle счётчик сброшен
|
||
assert fetcher._page_count == 0
|
||
|
||
|
||
# ── crash-recovery ─────────────────────────────────────────────────────────────
|
||
|
||
|
||
class _FakeTargetClosedError(Exception):
|
||
"""Имитирует playwright TargetClosedError (проверка по имени класса)."""
|
||
|
||
|
||
_FakeTargetClosedError.__name__ = "TargetClosedError"
|
||
_FakeTargetClosedError.__qualname__ = "TargetClosedError"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_retries_once_on_target_closed_error():
|
||
"""fetch() при TargetClosedError перезапускает браузер и повторяет запрос."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
html = "<html>retry-ok</html>"
|
||
|
||
# Первая страница бросает краш, вторая отвечает нормально
|
||
crash_page = MagicMock()
|
||
crash_page.goto = AsyncMock(side_effect=_FakeTargetClosedError("target closed"))
|
||
crash_page.close = AsyncMock()
|
||
|
||
ok_page = _make_mock_page(html=html)
|
||
browser = _make_mock_browser()
|
||
|
||
call_num = 0
|
||
|
||
async def new_page_side_effect() -> MagicMock:
|
||
nonlocal call_num
|
||
call_num += 1
|
||
return crash_page if call_num == 1 else ok_page
|
||
|
||
browser.new_page = AsyncMock(side_effect=new_page_side_effect)
|
||
ctx = _make_mock_camoufox_ctx(browser)
|
||
|
||
ms = _mock_settings(wait_ms=0)
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
result = await fetcher.fetch("https://avito.ru/page")
|
||
|
||
assert result == html
|
||
# 2 обращения к new_page: первое (crash) + второе (retry после relaunch)
|
||
assert call_num == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_propagates_non_crash_exception():
|
||
"""fetch() не подавляет обычные (не-crash) исключения."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
page = MagicMock()
|
||
page.goto = AsyncMock(side_effect=ValueError("unexpected parse error"))
|
||
page.close = AsyncMock()
|
||
|
||
browser = _make_mock_browser(page=page)
|
||
ctx = _make_mock_camoufox_ctx(browser)
|
||
|
||
ms = _mock_settings(wait_ms=0)
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
with patch("camoufox.async_api.AsyncCamoufox", return_value=ctx):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
with pytest.raises(ValueError, match="unexpected parse error"):
|
||
await fetcher.fetch("https://avito.ru/bad")
|