bot-backend
d479edeb3f
All checks were successful
Deploy Trade-In / changes (push) Successful in 9s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / test (push) Successful in 1m25s
Deploy Trade-In / build-backend (push) Successful in 52s
Deploy Trade-In / build-browser (push) Successful in 2m53s
Deploy Trade-In / deploy (push) Successful in 1m30s
375 lines
15 KiB
Python
375 lines
15 KiB
Python
"""Tests for BrowserFetcher (#884/#905, HTTP-клиент).
|
||
|
||
Полностью замоканы через monkeypatch на httpx.AsyncClient.post:
|
||
не требуют установленного Firefox, camoufox или реального HTTP-сервера.
|
||
|
||
Покрывают:
|
||
- fetch(): POST к правильному endpoint + разбор {"html": "..."} из ответа
|
||
- HTTPError триггерит один retry, затем поднимает исключение
|
||
- URL строится из settings.browser_http_endpoint
|
||
- импорт модуля не ломается без playwright/camoufox
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from types import SimpleNamespace
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import httpx
|
||
import pytest
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
|
||
# ── helpers ────────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _mock_settings(
|
||
browser_http_endpoint: str = "http://fake-browser:3000",
|
||
) -> SimpleNamespace:
|
||
return SimpleNamespace(browser_http_endpoint=browser_http_endpoint)
|
||
|
||
|
||
def _make_ok_response(html: str = "<html>ok</html>") -> MagicMock:
|
||
"""Создаёт мок httpx.Response с JSON {"html": html} и статусом 200."""
|
||
resp = MagicMock(spec=httpx.Response)
|
||
resp.status_code = 200
|
||
resp.json = MagicMock(return_value={"html": html})
|
||
resp.raise_for_status = MagicMock() # нет исключения
|
||
return resp
|
||
|
||
|
||
def _make_ok_json_response(status: int = 200, body: str = "{}") -> MagicMock:
|
||
"""Создаёт мок httpx.Response с JSON {"status": ..., "body": ...} (для /fetch-json)."""
|
||
resp = MagicMock(spec=httpx.Response)
|
||
resp.status_code = 200
|
||
resp.json = MagicMock(return_value={"status": status, "body": body})
|
||
resp.raise_for_status = MagicMock() # нет исключения
|
||
return resp
|
||
|
||
|
||
def _make_error_response(status: int = 500) -> MagicMock:
|
||
"""Создаёт мок httpx.Response который кидает HTTPStatusError при raise_for_status."""
|
||
resp = MagicMock(spec=httpx.Response)
|
||
resp.status_code = status
|
||
resp.raise_for_status = MagicMock(
|
||
side_effect=httpx.HTTPStatusError(
|
||
f"Server error {status}",
|
||
request=MagicMock(),
|
||
response=resp,
|
||
)
|
||
)
|
||
return resp
|
||
|
||
|
||
# ── импорт не ломается ─────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_import_browser_fetcher() -> None:
|
||
"""import app.services.scrapers.browser_fetcher не поднимает исключений."""
|
||
import app.services.scrapers.browser_fetcher # noqa: F401
|
||
|
||
|
||
# ── fetch(): возвращает HTML из JSON-ответа ────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_returns_html_from_json_response() -> None:
|
||
"""fetch() разбирает {'html': ...} из 200-ответа и возвращает строку."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
expected_html = "<html><body>avito</body></html>"
|
||
ok_resp = _make_ok_response(html=expected_html)
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
fetcher._client.post = AsyncMock(return_value=ok_resp) # type: ignore[method-assign]
|
||
result = await fetcher.fetch("https://avito.ru/test")
|
||
|
||
assert result == expected_html
|
||
assert isinstance(result, str)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_posts_to_correct_endpoint() -> None:
|
||
"""fetch() делает POST к {browser_http_endpoint}/fetch с {"url", "source"}.
|
||
|
||
Дефолтный source="avito" — обратная совместимость со старым поведением, когда
|
||
сервер игнорирует source при выключенном FEATURE_BROWSER_POOL_ENABLED.
|
||
"""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
endpoint = "http://fake-browser:3000"
|
||
ok_resp = _make_ok_response()
|
||
ms = _mock_settings(browser_http_endpoint=endpoint)
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
post_mock = AsyncMock(return_value=ok_resp)
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
await fetcher.fetch("https://example.com/page")
|
||
|
||
post_mock.assert_called_once_with(
|
||
f"{endpoint}/fetch",
|
||
json={"url": "https://example.com/page", "source": "avito"},
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_posts_source_field_when_set() -> None:
|
||
"""BrowserFetcher(source="yandex")._post_fetch шлёт body с "source": "yandex"."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
endpoint = "http://fake-browser:3000"
|
||
ok_resp = _make_ok_response()
|
||
ms = _mock_settings(browser_http_endpoint=endpoint)
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher(source="yandex")
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
post_mock = AsyncMock(return_value=ok_resp)
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
await fetcher.fetch("https://realty.yandex.ru/ekb/")
|
||
|
||
post_mock.assert_called_once_with(
|
||
f"{endpoint}/fetch",
|
||
json={"url": "https://realty.yandex.ru/ekb/", "source": "yandex"},
|
||
)
|
||
|
||
|
||
# ── fetch_json(): in-page fetch через /fetch-json (#915 Stage 2) ───────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_json_returns_dict_from_json_response() -> None:
|
||
"""fetch_json() возвращает {"status","body"} dict из 200-ответа /fetch-json."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
ok_resp = _make_ok_json_response(status=200, body='{"items": []}')
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
fetcher._client.post = AsyncMock(return_value=ok_resp) # type: ignore[method-assign]
|
||
result = await fetcher.fetch_json("https://avito.ru/api/x")
|
||
|
||
assert result == {"status": 200, "body": '{"items": []}'}
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_json_posts_full_payload() -> None:
|
||
"""fetch_json() POST'ит url/source/method/headers/body/origin к /fetch-json."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
endpoint = "http://fake-browser:3000"
|
||
ok_resp = _make_ok_json_response()
|
||
ms = _mock_settings(browser_http_endpoint=endpoint)
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher(source="avito")
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
post_mock = AsyncMock(return_value=ok_resp)
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
await fetcher.fetch_json(
|
||
"https://avito.ru/api/listings",
|
||
method="POST",
|
||
headers={"content-type": "application/json"},
|
||
body='{"q": 1}',
|
||
origin="https://avito.ru/moskva/kvartiry",
|
||
)
|
||
|
||
post_mock.assert_called_once_with(
|
||
f"{endpoint}/fetch-json",
|
||
json={
|
||
"url": "https://avito.ru/api/listings",
|
||
"source": "avito",
|
||
"method": "POST",
|
||
"headers": {"content-type": "application/json"},
|
||
"body": '{"q": 1}',
|
||
"origin": "https://avito.ru/moskva/kvartiry",
|
||
},
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_json_defaults_get_empty_headers_none_body() -> None:
|
||
"""Дефолты: method=GET, headers={}, body=None, origin=None."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
endpoint = "http://fake-browser:3000"
|
||
ok_resp = _make_ok_json_response()
|
||
ms = _mock_settings(browser_http_endpoint=endpoint)
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher(source="cian")
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
post_mock = AsyncMock(return_value=ok_resp)
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
await fetcher.fetch_json("https://cian.ru/api/x")
|
||
|
||
post_mock.assert_called_once_with(
|
||
f"{endpoint}/fetch-json",
|
||
json={
|
||
"url": "https://cian.ru/api/x",
|
||
"source": "cian",
|
||
"method": "GET",
|
||
"headers": {},
|
||
"body": None,
|
||
"origin": None,
|
||
},
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_json_retries_once_on_transport_error() -> None:
|
||
"""TransportError первый раз → один retry → успешный второй вызов."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
ok_resp = _make_ok_json_response(status=200, body="retry-ok")
|
||
connect_err = httpx.ConnectError("Connection refused")
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
post_mock = AsyncMock(side_effect=[connect_err, ok_resp])
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
|
||
with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
|
||
result = await fetcher.fetch_json("https://avito.ru/api/retry")
|
||
|
||
assert result == {"status": 200, "body": "retry-ok"}
|
||
assert post_mock.await_count == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_json_raises_after_two_http_errors() -> None:
|
||
"""Оба вызова /fetch-json кидают HTTPError → исключение пробрасывается."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
error_resp_1 = _make_error_response(500)
|
||
error_resp_2 = _make_error_response(500)
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
post_mock = AsyncMock(side_effect=[error_resp_1, error_resp_2])
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
|
||
with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
|
||
with pytest.raises(httpx.HTTPStatusError):
|
||
await fetcher.fetch_json("https://avito.ru/api/fail")
|
||
|
||
assert post_mock.await_count == 2
|
||
|
||
|
||
# ── retry при HTTPError ────────────────────────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_retries_once_on_http_error() -> None:
|
||
"""При HTTPError первый вызов → retry → успешный второй вызов."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
html = "<html>retry-ok</html>"
|
||
ok_resp = _make_ok_response(html=html)
|
||
error_resp = _make_error_response(500)
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
# Первый вызов кидает HTTPStatusError, второй возвращает OK
|
||
post_mock = AsyncMock(side_effect=[error_resp, ok_resp])
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
|
||
with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
|
||
result = await fetcher.fetch("https://avito.ru/retry")
|
||
|
||
assert result == html
|
||
assert post_mock.await_count == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_raises_after_two_http_errors() -> None:
|
||
"""Если оба вызова кидают HTTPError — исключение пробрасывается."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
error_resp_1 = _make_error_response(500)
|
||
error_resp_2 = _make_error_response(500)
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
post_mock = AsyncMock(side_effect=[error_resp_1, error_resp_2])
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
|
||
with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
|
||
with pytest.raises(httpx.HTTPStatusError):
|
||
await fetcher.fetch("https://avito.ru/fail")
|
||
|
||
assert post_mock.await_count == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_retries_on_connect_error() -> None:
|
||
"""ConnectError (подкласс TransportError) тоже триггерит retry."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
html = "<html>connect-retry-ok</html>"
|
||
ok_resp = _make_ok_response(html=html)
|
||
connect_err = httpx.ConnectError("Connection refused")
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
post_mock = AsyncMock(side_effect=[connect_err, ok_resp])
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
|
||
with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()):
|
||
result = await fetcher.fetch("https://avito.ru/connect")
|
||
|
||
assert result == html
|
||
assert post_mock.await_count == 2
|
||
|
||
|
||
# ── non-http exception пробрасывается без перехвата ───────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_propagates_unexpected_exception() -> None:
|
||
"""Не-HTTP исключение (ValueError и т.п.) не подавляется и не вызывает retry."""
|
||
from app.services.scrapers.browser_fetcher import BrowserFetcher
|
||
|
||
ms = _mock_settings()
|
||
|
||
with patch("app.core.config.settings", ms):
|
||
fetcher = BrowserFetcher()
|
||
async with fetcher:
|
||
assert fetcher._client is not None
|
||
post_mock = AsyncMock(side_effect=ValueError("unexpected"))
|
||
fetcher._client.post = post_mock # type: ignore[method-assign]
|
||
|
||
with pytest.raises(ValueError, match="unexpected"):
|
||
await fetcher.fetch("https://avito.ru/bad")
|
||
|
||
assert post_mock.await_count == 1
|