bot-backend
b9eb478c8f
Route each scraper source (avito/cian/yandex/domclick) to its own camoufox browser+proxy so they no longer wedge each other through a single global egress. Feature-flagged (FEATURE_BROWSER_POOL_ENABLED, default OFF): with the flag off the /fetch and /login paths are byte-for-byte the existing single-browser behavior. When on, /fetch routes by body["source"] to a per-proxy browser via BROWSER_PROXY_MAP (BROWSER_PROXY_AVITO/CIAN/YANDEX/DOMCLICK with legacy fallbacks), each guarded by its own lazy-launched lock. /login stays single-browser in Phase 1. BrowserFetcher gains a source arg (default avito) and sends it in the /fetch body; all scraper callsites pass their source. No docker-compose/.env.runtime changes (Phase 2, owner-gated).
233 lines
9.4 KiB
Python
233 lines
9.4 KiB
Python
"""Tests for BrowserFetcher (#884/#905, HTTP-клиент).
|
||
|
||
Полностью замоканы через monkeypatch на httpx.AsyncClient.post:
|
||
не требуют установленного Firefox, camoufox или реального HTTP-сервера.
|
||
|
||
Покрывают:
|
||
- fetch(): POST к правильному endpoint + разбор {"html": "..."} из ответа
|
||
- HTTPError триггерит один retry, затем поднимает исключение
|
||
- URL строится из settings.browser_http_endpoint
|
||
- импорт модуля не ломается без playwright/camoufox
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from types import SimpleNamespace
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import httpx
|
||
import pytest
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
|
||
# ── helpers ────────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _mock_settings(
|
||
browser_http_endpoint: str = "http://fake-browser:3000",
|
||
) -> SimpleNamespace:
|
||
return SimpleNamespace(browser_http_endpoint=browser_http_endpoint)
|
||
|
||
|
||
def _make_ok_response(html: str = "<html>ok</html>") -> MagicMock:
|
||
"""Создаёт мок httpx.Response с JSON {"html": html} и статусом 200."""
|
||
resp = MagicMock(spec=httpx.Response)
|
||
resp.status_code = 200
|
||
resp.json = MagicMock(return_value={"html": html})
|
||
resp.raise_for_status = MagicMock() # нет исключения
|
||
return resp
|
||
|
||
|
||
def _make_error_response(status: int = 500) -> MagicMock:
|
||
"""Создаёт мок httpx.Response который кидает HTTPStatusError при raise_for_status."""
|
||
resp = MagicMock(spec=httpx.Response)
|
||
resp.status_code = status
|
||
resp.raise_for_status = MagicMock(
|
||
side_effect=httpx.HTTPStatusError(
|
||
f"Server error {status}",
|
||
request=MagicMock(),
|
||
response=resp,
|
||
)
|
||
)
|
||
return resp
|
||
|
||
|
||
# ── импорт не ломается ─────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_import_browser_fetcher() -> None:
|
||
"""import app.services.scrapers.browser_fetcher не поднимает исключений."""
|
||
import app.services.scrapers.browser_fetcher # noqa: F401
|
||
|
||
|
||
# ── fetch(): возвращает HTML из JSON-ответа ────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_returns_html_from_json_response() -> None:
|
||
"""fetch() разбирает {'html': ...} из 200-ответа и возвращает строку."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
expected_html = "<html><body>avito</body></html>"
|
||
ok_resp = _make_ok_response(html=expected_html)
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
fetcher._client.post = AsyncMock(return_value=ok_resp) # type: ignore[method-assign]
|
||
result = await fetcher.fetch("https://avito.ru/test")
|
||
|
||
assert result == expected_html
|
||
assert isinstance(result, str)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_posts_to_correct_endpoint() -> None:
|
||
"""fetch() делает POST к {browser_http_endpoint}/fetch с {"url", "source"}.
|
||
|
||
Дефолтный source="avito" — обратная совместимость со старым поведением, когда
|
||
сервер игнорирует source при выключенном FEATURE_BROWSER_POOL_ENABLED.
|
||
"""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
endpoint = "http://fake-browser:3000"
|
||
ok_resp = _make_ok_response()
|
||
ms = _mock_settings(browser_http_endpoint=endpoint)
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
post_mock = AsyncMock(return_value=ok_resp)
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
await fetcher.fetch("https://example.com/page")
|
||
|
||
post_mock.assert_called_once_with(
|
||
f"{endpoint}/fetch",
|
||
json={"url": "https://example.com/page", "source": "avito"},
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_posts_source_field_when_set() -> None:
|
||
"""BrowserFetcher(source="yandex")._post_fetch шлёт body с "source": "yandex"."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
endpoint = "http://fake-browser:3000"
|
||
ok_resp = _make_ok_response()
|
||
ms = _mock_settings(browser_http_endpoint=endpoint)
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher(source="yandex")
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
post_mock = AsyncMock(return_value=ok_resp)
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
await fetcher.fetch("https://realty.yandex.ru/ekb/")
|
||
|
||
post_mock.assert_called_once_with(
|
||
f"{endpoint}/fetch",
|
||
json={"url": "https://realty.yandex.ru/ekb/", "source": "yandex"},
|
||
)
|
||
|
||
|
||
# ── retry при HTTPError ────────────────────────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_retries_once_on_http_error() -> None:
|
||
"""При HTTPError первый вызов → retry → успешный второй вызов."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
html = "<html>retry-ok</html>"
|
||
ok_resp = _make_ok_response(html=html)
|
||
error_resp = _make_error_response(500)
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
# Первый вызов кидает HTTPStatusError, второй возвращает OK
|
||
post_mock = AsyncMock(side_effect=[error_resp, ok_resp])
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
|
||
with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
|
||
result = await fetcher.fetch("https://avito.ru/retry")
|
||
|
||
assert result == html
|
||
assert post_mock.await_count == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_raises_after_two_http_errors() -> None:
|
||
"""Если оба вызова кидают HTTPError — исключение пробрасывается."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
error_resp_1 = _make_error_response(500)
|
||
error_resp_2 = _make_error_response(500)
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
post_mock = AsyncMock(side_effect=[error_resp_1, error_resp_2])
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
|
||
with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
|
||
with pytest.raises(httpx.HTTPStatusError):
|
||
await fetcher.fetch("https://avito.ru/fail")
|
||
|
||
assert post_mock.await_count == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_retries_on_connect_error() -> None:
|
||
"""ConnectError (подкласс TransportError) тоже триггерит retry."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
html = "<html>connect-retry-ok</html>"
|
||
ok_resp = _make_ok_response(html=html)
|
||
connect_err = httpx.ConnectError("Connection refused")
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
post_mock = AsyncMock(side_effect=[connect_err, ok_resp])
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
|
||
with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
|
||
result = await fetcher.fetch("https://avito.ru/connect")
|
||
|
||
assert result == html
|
||
assert post_mock.await_count == 2
|
||
|
||
|
||
# ── non-http exception пробрасывается без перехвата ───────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_propagates_unexpected_exception() -> None:
|
||
"""Не-HTTP исключение (ValueError и т.п.) не подавляется и не вызывает retry."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
post_mock = AsyncMock(side_effect=ValueError("unexpected"))
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
|
||
with pytest.raises(ValueError, match="unexpected"):
|
||
await fetcher.fetch("https://avito.ru/bad")
|
||
|
||
assert post_mock.await_count == 1
|