bot-backend
6044a1fe3c
The playwright WS-server path (camoufox.server.launch_server) is incompatible
with playwright >=1.45: camoufox 0.4.11's launchServer.js requires
playwright/driver/package/lib/browserServerImpl.js, which no longer exists in
any playwright 1.45-1.60 → tradein-browser crash-looped with MODULE_NOT_FOUND
('Server process terminated unexpectedly'). Verified by running the built image.
Pivot (no playwright protocol between containers):
- browser/server.py: aiohttp service holding a local AsyncCamoufox (headless,
os/locale/geoip/humanize, proxy from SCRAPER_PROXY_URL). POST /fetch {url} →
new_page/goto/content/close → {html}; GET /health. Page-recycle every N +
crash-recovery (relaunch + 1 retry), serialized via asyncio.Lock.
- browser/Dockerfile: +aiohttp (camoufox fetch-as-root/#908 block untouched).
- backend BrowserFetcher: playwright.connect → httpx POST to the browser service
(one retry on transport/HTTP error). All playwright imports dropped.
- config: browser_ws_endpoint → browser_http_endpoint (http://tradein-browser:3000).
Verified by a REAL local image build + run: image builds, container stays Up
(Restarts=0, no crash-loop), GET /health=200, POST /fetch launches camoufox and
drives Firefox (only NS_ERROR_UNKNOWN_HOST = no DNS egress in the local build env,
proving the full HTTP→camoufox→Firefox chain; real fetch validates on prod with
network+proxy). 7 unit tests + ruff clean.
Dormant: scraper_fetch_mode stays curl_cffi. Refs #905, #883, #884
206 lines
8.3 KiB
Python
206 lines
8.3 KiB
Python
"""Tests for BrowserFetcher (#884/#905, HTTP-клиент).
|
||
|
||
Полностью замоканы через monkeypatch на httpx.AsyncClient.post:
|
||
не требуют установленного Firefox, camoufox или реального HTTP-сервера.
|
||
|
||
Покрывают:
|
||
- fetch(): POST к правильному endpoint + разбор {"html": "..."} из ответа
|
||
- HTTPError триггерит один retry, затем поднимает исключение
|
||
- URL строится из settings.browser_http_endpoint
|
||
- импорт модуля не ломается без playwright/camoufox
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from types import SimpleNamespace
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import httpx
|
||
import pytest
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
|
||
# ── helpers ────────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _mock_settings(
|
||
browser_http_endpoint: str = "http://fake-browser:3000",
|
||
) -> SimpleNamespace:
|
||
return SimpleNamespace(browser_http_endpoint=browser_http_endpoint)
|
||
|
||
|
||
def _make_ok_response(html: str = "<html>ok</html>") -> MagicMock:
|
||
"""Создаёт мок httpx.Response с JSON {"html": html} и статусом 200."""
|
||
resp = MagicMock(spec=httpx.Response)
|
||
resp.status_code = 200
|
||
resp.json = MagicMock(return_value={"html": html})
|
||
resp.raise_for_status = MagicMock() # нет исключения
|
||
return resp
|
||
|
||
|
||
def _make_error_response(status: int = 500) -> MagicMock:
|
||
"""Создаёт мок httpx.Response который кидает HTTPStatusError при raise_for_status."""
|
||
resp = MagicMock(spec=httpx.Response)
|
||
resp.status_code = status
|
||
resp.raise_for_status = MagicMock(
|
||
side_effect=httpx.HTTPStatusError(
|
||
f"Server error {status}",
|
||
request=MagicMock(),
|
||
response=resp,
|
||
)
|
||
)
|
||
return resp
|
||
|
||
|
||
# ── импорт не ломается ─────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_import_browser_fetcher() -> None:
|
||
"""import app.services.scrapers.browser_fetcher не поднимает исключений."""
|
||
import app.services.scrapers.browser_fetcher # noqa: F401
|
||
|
||
|
||
# ── fetch(): возвращает HTML из JSON-ответа ────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_returns_html_from_json_response() -> None:
|
||
"""fetch() разбирает {'html': ...} из 200-ответа и возвращает строку."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
expected_html = "<html><body>avito</body></html>"
|
||
ok_resp = _make_ok_response(html=expected_html)
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
fetcher._client.post = AsyncMock(return_value=ok_resp) # type: ignore[method-assign]
|
||
result = await fetcher.fetch("https://avito.ru/test")
|
||
|
||
assert result == expected_html
|
||
assert isinstance(result, str)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_posts_to_correct_endpoint() -> None:
|
||
"""fetch() делает POST к {browser_http_endpoint}/fetch с {"url": ...}."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
endpoint = "http://fake-browser:3000"
|
||
ok_resp = _make_ok_response()
|
||
ms = _mock_settings(browser_http_endpoint=endpoint)
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
post_mock = AsyncMock(return_value=ok_resp)
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
await fetcher.fetch("https://example.com/page")
|
||
|
||
post_mock.assert_called_once_with(
|
||
f"{endpoint}/fetch",
|
||
json={"url": "https://example.com/page"},
|
||
)
|
||
|
||
|
||
# ── retry при HTTPError ────────────────────────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_retries_once_on_http_error() -> None:
|
||
"""При HTTPError первый вызов → retry → успешный второй вызов."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
html = "<html>retry-ok</html>"
|
||
ok_resp = _make_ok_response(html=html)
|
||
error_resp = _make_error_response(500)
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
# Первый вызов кидает HTTPStatusError, второй возвращает OK
|
||
post_mock = AsyncMock(side_effect=[error_resp, ok_resp])
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
|
||
with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
|
||
result = await fetcher.fetch("https://avito.ru/retry")
|
||
|
||
assert result == html
|
||
assert post_mock.await_count == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_raises_after_two_http_errors() -> None:
|
||
"""Если оба вызова кидают HTTPError — исключение пробрасывается."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
error_resp_1 = _make_error_response(500)
|
||
error_resp_2 = _make_error_response(500)
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
post_mock = AsyncMock(side_effect=[error_resp_1, error_resp_2])
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
|
||
with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
|
||
with pytest.raises(httpx.HTTPStatusError):
|
||
await fetcher.fetch("https://avito.ru/fail")
|
||
|
||
assert post_mock.await_count == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_retries_on_connect_error() -> None:
|
||
"""ConnectError (подкласс TransportError) тоже триггерит retry."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
html = "<html>connect-retry-ok</html>"
|
||
ok_resp = _make_ok_response(html=html)
|
||
connect_err = httpx.ConnectError("Connection refused")
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
post_mock = AsyncMock(side_effect=[connect_err, ok_resp])
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
|
||
with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
|
||
result = await fetcher.fetch("https://avito.ru/connect")
|
||
|
||
assert result == html
|
||
assert post_mock.await_count == 2
|
||
|
||
|
||
# ── non-http exception пробрасывается без перехвата ───────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_propagates_unexpected_exception() -> None:
|
||
"""Не-HTTP исключение (ValueError и т.п.) не подавляется и не вызывает retry."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
post_mock = AsyncMock(side_effect=ValueError("unexpected"))
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
|
||
with pytest.raises(ValueError, match="unexpected"):
|
||
await fetcher.fetch("https://avito.ru/bad")
|
||
|
||
assert post_mock.await_count == 1
|