"""Tests for BrowserFetcher (#884/#905, HTTP-клиент). Полностью замоканы через monkeypatch на httpx.AsyncClient.post: не требуют установленного Firefox, camoufox или реального HTTP-сервера. Покрывают: - fetch(): POST к правильному endpoint + разбор {"html": "..."} из ответа - HTTPError триггерит один retry, затем поднимает исключение - URL строится из settings.browser_http_endpoint - импорт модуля не ломается без playwright/camoufox """ from __future__ import annotations import os from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") # ── helpers ──────────────────────────────────────────────────────────────────── def _mock_settings( browser_http_endpoint: str = "http://fake-browser:3000", ) -> SimpleNamespace: return SimpleNamespace(browser_http_endpoint=browser_http_endpoint) def _make_ok_response(html: str = "ok") -> MagicMock: """Создаёт мок httpx.Response с JSON {"html": html} и статусом 200.""" resp = MagicMock(spec=httpx.Response) resp.status_code = 200 resp.json = MagicMock(return_value={"html": html}) resp.raise_for_status = MagicMock() # нет исключения return resp def _make_ok_json_response(status: int = 200, body: str = "{}") -> MagicMock: """Создаёт мок httpx.Response с JSON {"status": ..., "body": ...} (для /fetch-json).""" resp = MagicMock(spec=httpx.Response) resp.status_code = 200 resp.json = MagicMock(return_value={"status": status, "body": body}) resp.raise_for_status = MagicMock() # нет исключения return resp def _make_error_response(status: int = 500) -> MagicMock: """Создаёт мок httpx.Response который кидает HTTPStatusError при raise_for_status.""" resp = MagicMock(spec=httpx.Response) resp.status_code = status resp.raise_for_status = MagicMock( side_effect=httpx.HTTPStatusError( f"Server error {status}", request=MagicMock(), response=resp, ) ) return resp # ── импорт не ломается ───────────────────────────────────────────────────────── def test_import_browser_fetcher() -> None: """import app.services.scrapers.browser_fetcher не поднимает исключений.""" import app.services.scrapers.browser_fetcher # noqa: F401 # ── fetch(): возвращает HTML из JSON-ответа ──────────────────────────────────── @pytest.mark.asyncio async def test_fetch_returns_html_from_json_response() -> None: """fetch() разбирает {'html': ...} из 200-ответа и возвращает строку.""" from app.services.scrapers.browser_fetcher import BrowserFetcher expected_html = "avito" ok_resp = _make_ok_response(html=expected_html) ms = _mock_settings() with patch("app.core.config.settings", ms): fetcher = BrowserFetcher() async with fetcher: assert fetcher._client is not None fetcher._client.post = AsyncMock(return_value=ok_resp) # type: ignore[method-assign] result = await fetcher.fetch("https://avito.ru/test") assert result == expected_html assert isinstance(result, str) @pytest.mark.asyncio async def test_fetch_posts_to_correct_endpoint() -> None: """fetch() делает POST к {browser_http_endpoint}/fetch с {"url", "source"}. Дефолтный source="avito" — обратная совместимость со старым поведением, когда сервер игнорирует source при выключенном FEATURE_BROWSER_POOL_ENABLED. """ from app.services.scrapers.browser_fetcher import BrowserFetcher endpoint = "http://fake-browser:3000" ok_resp = _make_ok_response() ms = _mock_settings(browser_http_endpoint=endpoint) with patch("app.core.config.settings", ms): fetcher = BrowserFetcher() async with fetcher: assert fetcher._client is not None post_mock = AsyncMock(return_value=ok_resp) fetcher._client.post = post_mock # type: ignore[method-assign] await fetcher.fetch("https://example.com/page") post_mock.assert_called_once_with( f"{endpoint}/fetch", json={"url": "https://example.com/page", "source": "avito"}, ) @pytest.mark.asyncio async def test_fetch_posts_source_field_when_set() -> None: """BrowserFetcher(source="yandex")._post_fetch шлёт body с "source": "yandex".""" from app.services.scrapers.browser_fetcher import BrowserFetcher endpoint = "http://fake-browser:3000" ok_resp = _make_ok_response() ms = _mock_settings(browser_http_endpoint=endpoint) with patch("app.core.config.settings", ms): fetcher = BrowserFetcher(source="yandex") async with fetcher: assert fetcher._client is not None post_mock = AsyncMock(return_value=ok_resp) fetcher._client.post = post_mock # type: ignore[method-assign] await fetcher.fetch("https://realty.yandex.ru/ekb/") post_mock.assert_called_once_with( f"{endpoint}/fetch", json={"url": "https://realty.yandex.ru/ekb/", "source": "yandex"}, ) # ── fetch_json(): in-page fetch через /fetch-json (#915 Stage 2) ─────────────── @pytest.mark.asyncio async def test_fetch_json_returns_dict_from_json_response() -> None: """fetch_json() возвращает {"status","body"} dict из 200-ответа /fetch-json.""" from app.services.scrapers.browser_fetcher import BrowserFetcher ok_resp = _make_ok_json_response(status=200, body='{"items": []}') ms = _mock_settings() with patch("app.core.config.settings", ms): fetcher = BrowserFetcher() async with fetcher: assert fetcher._client is not None fetcher._client.post = AsyncMock(return_value=ok_resp) # type: ignore[method-assign] result = await fetcher.fetch_json("https://avito.ru/api/x") assert result == {"status": 200, "body": '{"items": []}'} @pytest.mark.asyncio async def test_fetch_json_posts_full_payload() -> None: """fetch_json() POST'ит url/source/method/headers/body/origin к /fetch-json.""" from app.services.scrapers.browser_fetcher import BrowserFetcher endpoint = "http://fake-browser:3000" ok_resp = _make_ok_json_response() ms = _mock_settings(browser_http_endpoint=endpoint) with patch("app.core.config.settings", ms): fetcher = BrowserFetcher(source="avito") async with fetcher: assert fetcher._client is not None post_mock = AsyncMock(return_value=ok_resp) fetcher._client.post = post_mock # type: ignore[method-assign] await fetcher.fetch_json( "https://avito.ru/api/listings", method="POST", headers={"content-type": "application/json"}, body='{"q": 1}', origin="https://avito.ru/moskva/kvartiry", ) post_mock.assert_called_once_with( f"{endpoint}/fetch-json", json={ "url": "https://avito.ru/api/listings", "source": "avito", "method": "POST", "headers": {"content-type": "application/json"}, "body": '{"q": 1}', "origin": "https://avito.ru/moskva/kvartiry", }, ) @pytest.mark.asyncio async def test_fetch_json_defaults_get_empty_headers_none_body() -> None: """Дефолты: method=GET, headers={}, body=None, origin=None.""" from app.services.scrapers.browser_fetcher import BrowserFetcher endpoint = "http://fake-browser:3000" ok_resp = _make_ok_json_response() ms = _mock_settings(browser_http_endpoint=endpoint) with patch("app.core.config.settings", ms): fetcher = BrowserFetcher(source="cian") async with fetcher: assert fetcher._client is not None post_mock = AsyncMock(return_value=ok_resp) fetcher._client.post = post_mock # type: ignore[method-assign] await fetcher.fetch_json("https://cian.ru/api/x") post_mock.assert_called_once_with( f"{endpoint}/fetch-json", json={ "url": "https://cian.ru/api/x", "source": "cian", "method": "GET", "headers": {}, "body": None, "origin": None, }, ) @pytest.mark.asyncio async def test_fetch_json_retries_once_on_transport_error() -> None: """TransportError первый раз → один retry → успешный второй вызов.""" from app.services.scrapers.browser_fetcher import BrowserFetcher ok_resp = _make_ok_json_response(status=200, body="retry-ok") connect_err = httpx.ConnectError("Connection refused") ms = _mock_settings() with patch("app.core.config.settings", ms): fetcher = BrowserFetcher() async with fetcher: assert fetcher._client is not None post_mock = AsyncMock(side_effect=[connect_err, ok_resp]) fetcher._client.post = post_mock # type: ignore[method-assign] with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()): result = await fetcher.fetch_json("https://avito.ru/api/retry") assert result == {"status": 200, "body": "retry-ok"} assert post_mock.await_count == 2 @pytest.mark.asyncio async def test_fetch_json_raises_after_two_http_errors() -> None: """Оба вызова /fetch-json кидают HTTPError → исключение пробрасывается.""" from app.services.scrapers.browser_fetcher import BrowserFetcher error_resp_1 = _make_error_response(500) error_resp_2 = _make_error_response(500) ms = _mock_settings() with patch("app.core.config.settings", ms): fetcher = BrowserFetcher() async with fetcher: assert fetcher._client is not None post_mock = AsyncMock(side_effect=[error_resp_1, error_resp_2]) fetcher._client.post = post_mock # type: ignore[method-assign] with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()): with pytest.raises(httpx.HTTPStatusError): await fetcher.fetch_json("https://avito.ru/api/fail") assert post_mock.await_count == 2 # ── retry при HTTPError ──────────────────────────────────────────────────────── @pytest.mark.asyncio async def test_fetch_retries_once_on_http_error() -> None: """При HTTPError первый вызов → retry → успешный второй вызов.""" from app.services.scrapers.browser_fetcher import BrowserFetcher html = "retry-ok" ok_resp = _make_ok_response(html=html) error_resp = _make_error_response(500) ms = _mock_settings() with patch("app.core.config.settings", ms): fetcher = BrowserFetcher() async with fetcher: assert fetcher._client is not None # Первый вызов кидает HTTPStatusError, второй возвращает OK post_mock = AsyncMock(side_effect=[error_resp, ok_resp]) fetcher._client.post = post_mock # type: ignore[method-assign] with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()): result = await fetcher.fetch("https://avito.ru/retry") assert result == html assert post_mock.await_count == 2 @pytest.mark.asyncio async def test_fetch_raises_after_two_http_errors() -> None: """Если оба вызова кидают HTTPError — исключение пробрасывается.""" from app.services.scrapers.browser_fetcher import BrowserFetcher error_resp_1 = _make_error_response(500) error_resp_2 = _make_error_response(500) ms = _mock_settings() with patch("app.core.config.settings", ms): fetcher = BrowserFetcher() async with fetcher: assert fetcher._client is not None post_mock = AsyncMock(side_effect=[error_resp_1, error_resp_2]) fetcher._client.post = post_mock # type: ignore[method-assign] with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()): with pytest.raises(httpx.HTTPStatusError): await fetcher.fetch("https://avito.ru/fail") assert post_mock.await_count == 2 @pytest.mark.asyncio async def test_fetch_retries_on_connect_error() -> None: """ConnectError (подкласс TransportError) тоже триггерит retry.""" from app.services.scrapers.browser_fetcher import BrowserFetcher html = "connect-retry-ok" ok_resp = _make_ok_response(html=html) connect_err = httpx.ConnectError("Connection refused") ms = _mock_settings() with patch("app.core.config.settings", ms): fetcher = BrowserFetcher() async with fetcher: assert fetcher._client is not None post_mock = AsyncMock(side_effect=[connect_err, ok_resp]) fetcher._client.post = post_mock # type: ignore[method-assign] with patch("app.services.scrapers.browser_fetcher.asyncio.sleep", AsyncMock()): result = await fetcher.fetch("https://avito.ru/connect") assert result == html assert post_mock.await_count == 2 # ── non-http exception пробрасывается без перехвата ─────────────────────────── @pytest.mark.asyncio async def test_fetch_propagates_unexpected_exception() -> None: """Не-HTTP исключение (ValueError и т.п.) не подавляется и не вызывает retry.""" from app.services.scrapers.browser_fetcher import BrowserFetcher ms = _mock_settings() with patch("app.core.config.settings", ms): fetcher = BrowserFetcher() async with fetcher: assert fetcher._client is not None post_mock = AsyncMock(side_effect=ValueError("unexpected")) fetcher._client.post = post_mock # type: ignore[method-assign] with pytest.raises(ValueError, match="unexpected"): await fetcher.fetch("https://avito.ru/bad") assert post_mock.await_count == 1