bot-backend
5fb367295a
Phase 0.5 — revise #884's in-backend camoufox into a dedicated browser service. camoufox.server.launch_server exposes a Playwright WS endpoint (fingerprint/geoip/ proxy configured server-side); the backend connects remotely via playwright, so it no longer ships Firefox. - NEW tradein-mvp/browser/{server.py,Dockerfile}: camoufox WS server on :3000; Firefox fetch under app-user + ENV HOME (carries the #899 cache-path fix) - BrowserFetcher rewritten launch-in-process → playwright.firefox.connect(ws_endpoint); recycle (close+reconnect every N) + crash-recovery (TargetClosed → reconnect + 1 retry) - backend Dockerfile slimmed: drop 5 Firefox libs + camoufox fetch layer - pyproject: remove camoufox[geoip] from backend, promote playwright to main deps - docker-compose.prod.yml: add browser service (mem_limit 2.5g moves here), remove mem_limit from backend, backend depends_on browser (service_started) - config: browser_ws_endpoint default ws://tradein-browser:3000/playwright - tests/scrapers/test_browser_fetcher.py rewritten for connect-mode (6 pass) Dormant: scraper_fetch_mode stays curl_cffi → no prod behavior change. launch_server signature verified (**kwargs→launch_options, port passes through). Refs #905, #883, #884, #899
263 lines
9.6 KiB
Python
263 lines
9.6 KiB
Python
"""Tests for BrowserFetcher (#884/#905, connect-mode).
|
||
|
||
Полностью замоканы: не требуют установленного Firefox, camoufox или реального
|
||
WS-сервера. Покрывают:
|
||
- fetch(): new_page → goto → wait_for_timeout → content → close
|
||
- recycle: счётчик страниц → reconnect при достижении порога
|
||
- crash-recovery: TargetClosedError → reconnect → retry
|
||
- page counter сбрасывается после reconnect
|
||
- non-crash exception пробрасывается без перехвата
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from types import SimpleNamespace
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
|
||
# ── helpers ────────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _mock_settings(
|
||
proxy_url: str | None = None,
|
||
recycle_pages: int = 15,
|
||
nav_timeout_ms: int = 60000,
|
||
wait_ms: int = 2500,
|
||
ws_endpoint: str = "ws://fake-browser:3000/playwright",
|
||
) -> SimpleNamespace:
|
||
return SimpleNamespace(
|
||
scraper_proxy_url=proxy_url,
|
||
browser_recycle_pages=recycle_pages,
|
||
browser_nav_timeout_ms=nav_timeout_ms,
|
||
browser_wait_ms=wait_ms,
|
||
browser_ws_endpoint=ws_endpoint,
|
||
)
|
||
|
||
|
||
def _make_mock_page(html: str = "<html>ok</html>") -> MagicMock:
|
||
"""Создаёт мок страницы playwright с нужными async-методами."""
|
||
page = MagicMock()
|
||
page.goto = AsyncMock()
|
||
page.wait_for_timeout = AsyncMock()
|
||
page.content = AsyncMock(return_value=html)
|
||
page.close = AsyncMock()
|
||
return page
|
||
|
||
|
||
def _make_playwright_stack(
|
||
page: MagicMock | None = None,
|
||
html: str = "<html>ok</html>",
|
||
) -> tuple[MagicMock, MagicMock, MagicMock]:
|
||
"""Создаёт полную mock-цепочку playwright: pw_cm → pw → browser → page.
|
||
|
||
Returns:
|
||
(pw_cm, pw_mock, browser_mock)
|
||
"""
|
||
if page is None:
|
||
page = _make_mock_page(html=html)
|
||
|
||
browser_mock = MagicMock()
|
||
browser_mock.new_page = AsyncMock(return_value=page)
|
||
browser_mock.close = AsyncMock()
|
||
|
||
firefox_mock = MagicMock()
|
||
firefox_mock.connect = AsyncMock(return_value=browser_mock)
|
||
|
||
pw_mock = MagicMock()
|
||
pw_mock.firefox = firefox_mock
|
||
pw_mock.stop = AsyncMock()
|
||
|
||
async def _start() -> MagicMock:
|
||
return pw_mock
|
||
|
||
pw_cm = MagicMock()
|
||
pw_cm.start = _start
|
||
|
||
return pw_cm, pw_mock, browser_mock
|
||
|
||
|
||
# ── fetch(): new_page → goto → content → close ────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_calls_page_lifecycle():
|
||
"""fetch() вызывает new_page, goto, wait_for_timeout, content, close."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
expected_html = "<html><body>test</body></html>"
|
||
page = _make_mock_page(html=expected_html)
|
||
pw_cm, _pw, browser = _make_playwright_stack(page=page)
|
||
ms = _mock_settings(recycle_pages=15, nav_timeout_ms=60000, wait_ms=2500)
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
with patch("playwright.async_api.async_playwright", return_value=pw_cm):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
result = await fetcher.fetch("https://example.com")
|
||
|
||
assert result == expected_html
|
||
browser.new_page.assert_called_once()
|
||
page.goto.assert_called_once_with(
|
||
"https://example.com", timeout=60000, wait_until="domcontentloaded"
|
||
)
|
||
page.wait_for_timeout.assert_called_once_with(2500)
|
||
page.content.assert_called_once()
|
||
page.close.assert_called_once()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_returns_html_string():
|
||
"""fetch() возвращает строку HTML от page.content()."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
html = "<html><title>Avito</title></html>"
|
||
pw_cm, _pw, _browser = _make_playwright_stack(html=html)
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
with patch("playwright.async_api.async_playwright", return_value=pw_cm):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
result = await fetcher.fetch("https://avito.ru/test")
|
||
|
||
assert result == html
|
||
assert isinstance(result, str)
|
||
|
||
|
||
# ── recycle: reconnect at threshold ───────────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_recycle_triggers_reconnect_at_threshold():
|
||
"""После recycle_pages страниц выполняется переподключение (_reconnect)."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
recycle_after = 3
|
||
pw_cm, _pw, _browser = _make_playwright_stack()
|
||
ms = _mock_settings(recycle_pages=recycle_after, wait_ms=0)
|
||
|
||
reconnect_count = 0
|
||
original_reconnect = BrowserFetcher._reconnect
|
||
|
||
async def counting_reconnect(self: BrowserFetcher) -> None:
|
||
nonlocal reconnect_count
|
||
reconnect_count += 1
|
||
await original_reconnect(self)
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
with patch("playwright.async_api.async_playwright", return_value=pw_cm):
|
||
with patch.object(BrowserFetcher, "_reconnect", counting_reconnect):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert reconnect_count == 0
|
||
for _ in range(recycle_after):
|
||
await fetcher.fetch("https://example.com")
|
||
# Третья страница триггерит recycle
|
||
assert reconnect_count == 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_page_counter_resets_after_reconnect():
|
||
"""После recycle _page_count сбрасывается в 0."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
recycle_after = 2
|
||
pw_cm, _pw, _browser = _make_playwright_stack()
|
||
ms = _mock_settings(recycle_pages=recycle_after, wait_ms=0)
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
with patch("playwright.async_api.async_playwright", return_value=pw_cm):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
for _ in range(recycle_after):
|
||
await fetcher.fetch("https://example.com")
|
||
# После recycle счётчик сброшен
|
||
assert fetcher._page_count == 0
|
||
|
||
|
||
# ── crash-recovery ─────────────────────────────────────────────────────────────
|
||
|
||
|
||
class _FakeTargetClosedError(Exception):
|
||
"""Имитирует playwright TargetClosedError (проверка по имени класса)."""
|
||
|
||
|
||
_FakeTargetClosedError.__name__ = "TargetClosedError"
|
||
_FakeTargetClosedError.__qualname__ = "TargetClosedError"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_retries_once_on_target_closed_error():
|
||
"""fetch() при TargetClosedError переподключается и повторяет запрос."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
html = "<html>retry-ok</html>"
|
||
|
||
# Первая страница кидает краш, вторая отвечает нормально
|
||
crash_page = MagicMock()
|
||
crash_page.goto = AsyncMock(side_effect=_FakeTargetClosedError("target closed"))
|
||
crash_page.close = AsyncMock()
|
||
|
||
ok_page = _make_mock_page(html=html)
|
||
|
||
browser_mock = MagicMock()
|
||
browser_mock.close = AsyncMock()
|
||
call_num = 0
|
||
|
||
async def new_page_side_effect() -> MagicMock:
|
||
nonlocal call_num
|
||
call_num += 1
|
||
return crash_page if call_num == 1 else ok_page
|
||
|
||
browser_mock.new_page = AsyncMock(side_effect=new_page_side_effect)
|
||
|
||
firefox_mock = MagicMock()
|
||
firefox_mock.connect = AsyncMock(return_value=browser_mock)
|
||
|
||
pw_mock = MagicMock()
|
||
pw_mock.firefox = firefox_mock
|
||
pw_mock.stop = AsyncMock()
|
||
|
||
async def _start() -> MagicMock:
|
||
return pw_mock
|
||
|
||
pw_cm = MagicMock()
|
||
pw_cm.start = _start
|
||
|
||
ms = _mock_settings(wait_ms=0)
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
with patch("playwright.async_api.async_playwright", return_value=pw_cm):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
result = await fetcher.fetch("https://avito.ru/page")
|
||
|
||
assert result == html
|
||
assert call_num == 2
|
||
# connect вызывался дважды: при __aenter__ + при reconnect
|
||
assert firefox_mock.connect.await_count == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_propagates_non_crash_exception():
|
||
"""fetch() не подавляет обычные (не-crash) исключения."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
page = MagicMock()
|
||
page.goto = AsyncMock(side_effect=ValueError("unexpected parse error"))
|
||
page.close = AsyncMock()
|
||
|
||
pw_cm, _pw, _browser = _make_playwright_stack(page=page)
|
||
ms = _mock_settings(wait_ms=0)
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
with patch("playwright.async_api.async_playwright", return_value=pw_cm):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
with pytest.raises(ValueError, match="unexpected parse error"):
|
||
await fetcher.fetch("https://avito.ru/bad")
|