"""Тесты browser-пула для kit `scraper_kit.browser_fetcher.BrowserFetcher` (#2164 P4). Инвариант ship-dark + fallback: - use_pool=False (дефолт) → в теле POST /fetch НЕТ поля "proxy"; proxy_provider не трогается (golden-parity: браузер юзает свой env-прокси BROWSER_PROXY_*). - use_pool=True + пул выдал lease → тело содержит "proxy"=lease.url + "proxy_kind"; на выходе mark_health(ok) + release (в finally — lease не течёт). - use_pool=True + пул пуст (acquire→None) → тела без "proxy", НЕ падаем. - use_pool=True + acquire бросил → fallback без "proxy", НЕ падаем. - fetch кинул → mark_health(ok=False) + release всё равно (finally). httpx полностью замокан: fetcher._client подменяется MagicMock'ом. """ from __future__ import annotations from typing import Any from unittest.mock import AsyncMock, MagicMock import pytest from scraper_kit.browser_fetcher import BrowserFetcher from scraper_kit.contracts import ProxyLease def _mock_client(json_payload: dict[str, Any], *, raise_exc: Exception | None = None) -> MagicMock: """httpx.AsyncClient-заглушка: .post → resp c raise_for_status/json.""" resp = MagicMock() if raise_exc is not None: resp.raise_for_status.side_effect = raise_exc else: resp.raise_for_status.return_value = None resp.json.return_value = json_payload client = MagicMock() client.post = AsyncMock(return_value=resp) return client class _FakeProxyProvider: """ProxyProvider-заглушка: acquire отдаёт заданный lease (или None), считает вызовы.""" def __init__(self, lease: ProxyLease | None, *, acquire_raises: bool = False) -> None: self._lease = lease self._acquire_raises = acquire_raises self.acquired: list[str] = [] self.released: list[int] = [] self.health: list[tuple[int, bool]] = [] def acquire(self, provider: str) -> ProxyLease | None: self.acquired.append(provider) if self._acquire_raises: raise RuntimeError("pool boom") return self._lease def release(self, lease: ProxyLease) -> None: self.released.append(lease.id) def mark_health(self, lease: ProxyLease, ok: bool, **_: Any) -> None: self.health.append((lease.id, ok)) def _fetcher(client: MagicMock, **kwargs: Any) -> BrowserFetcher: bf = BrowserFetcher(endpoint="http://browser:3000", **kwargs) bf._client = client return bf async def test_fetch_pool_off_no_proxy_in_body() -> None: """use_pool=False (дефолт) → тело без 'proxy', provider не трогается (parity).""" provider = _FakeProxyProvider(ProxyLease(id=1, url="http://pool:8080", kind="http")) client = _mock_client({"html": ""}) bf = _fetcher(client, source="avito", proxy_provider=provider) # use_pool default False html = await bf.fetch("https://avito.ru/x") assert html == "" body = client.post.call_args.kwargs["json"] assert "proxy" not in body assert provider.acquired == [] # пул не трогали async def test_fetch_pool_on_injects_proxy_and_releases() -> None: """use_pool=True + lease → 'proxy'/'proxy_kind' в теле; mark_health(True)+release.""" lease = ProxyLease(id=7, url="http://u:p@pool:8080", kind="http") provider = _FakeProxyProvider(lease) client = _mock_client({"html": ""}) bf = _fetcher(client, source="avito", proxy_provider=provider, use_pool=True) await bf.fetch("https://avito.ru/x") body = client.post.call_args.kwargs["json"] assert body["proxy"] == "http://u:p@pool:8080" assert body["proxy_kind"] == "http" assert provider.acquired == ["avito"] assert provider.health == [(7, True)] assert provider.released == [7] async def test_fetch_pool_empty_falls_back_no_proxy() -> None: """use_pool=True + пул пуст (acquire→None) → тело без 'proxy', без mark_health/release.""" provider = _FakeProxyProvider(None) client = _mock_client({"html": ""}) bf = _fetcher(client, source="cian", proxy_provider=provider, use_pool=True) await bf.fetch("https://cian.ru/x") body = client.post.call_args.kwargs["json"] assert "proxy" not in body assert provider.acquired == ["cian"] assert provider.released == [] assert provider.health == [] async def test_fetch_pool_acquire_error_falls_back() -> None: """acquire бросил → fallback без 'proxy', сбор НЕ падает.""" provider = _FakeProxyProvider(None, acquire_raises=True) client = _mock_client({"html": ""}) bf = _fetcher(client, source="yandex", proxy_provider=provider, use_pool=True) html = await bf.fetch("https://yandex.ru/x") assert html == "" body = client.post.call_args.kwargs["json"] assert "proxy" not in body assert provider.released == [] async def test_fetch_error_marks_health_false_and_releases() -> None: """fetch кинул → mark_health(ok=False) + release всё равно (finally, lease не течёт).""" lease = ProxyLease(id=3, url="http://pool:8080", kind="http") provider = _FakeProxyProvider(lease) # raise_for_status кидает НЕ-httpx ошибку → fetch() не ретраит, пробрасывает наверх. client = _mock_client({"html": "x"}, raise_exc=ValueError("bad status")) bf = _fetcher(client, source="avito", proxy_provider=provider, use_pool=True) with pytest.raises(ValueError): await bf.fetch("https://avito.ru/x") assert provider.health == [(3, False)] assert provider.released == [3] async def test_fetch_json_pool_on_injects_proxy() -> None: """fetch_json тоже прокидывает proxy из пула в тело /fetch-json.""" lease = ProxyLease(id=9, url="http://pool:8080", kind="http") provider = _FakeProxyProvider(lease) client = _mock_client({"status": 200, "body": "{}"}) bf = _fetcher(client, source="avito", proxy_provider=provider, use_pool=True) await bf.fetch_json( "https://avito.ru/api", method="POST", body="{}", origin="https://avito.ru/" ) body = client.post.call_args.kwargs["json"] assert body["proxy"] == "http://pool:8080" assert body["proxy_kind"] == "http" assert provider.released == [9] async def test_fetch_json_pool_off_no_proxy() -> None: """use_pool=False → fetch_json без 'proxy' в теле (parity).""" client = _mock_client({"status": 200, "body": "{}"}) bf = _fetcher(client, source="avito") await bf.fetch_json("https://avito.ru/api") body = client.post.call_args.kwargs["json"] assert "proxy" not in body # ── fetch() origin passthrough (DomClick SERP-anchor, зеркалит fetch_json) ─────── async def test_fetch_without_origin_sends_none() -> None: """fetch(url) без origin → payload содержит "origin": None (parity avito/cian/yandex). Ни один провайдер кроме domclick не передаёт origin — сервер (body.get("origin")) трактует None как «origin-goto не выполнять», поведение не меняется. """ client = _mock_client({"html": ""}) bf = _fetcher(client, source="avito") html = await bf.fetch("https://avito.ru/x") assert html == "" body = client.post.call_args.kwargs["json"] assert body["origin"] is None async def test_fetch_with_origin_sends_it_in_payload() -> None: """fetch(url, origin=X) → payload несёт origin=X (domclick vtorichka-SERP anchor).""" client = _mock_client({"html": ""}) bf = _fetcher(client, source="cian") await bf.fetch( "https://ekaterinburg.domclick.ru/card/sale__flat__1", origin="https://ekaterinburg.domclick.ru/pokupka/kvartiry/vtorichka", ) body = client.post.call_args.kwargs["json"] assert body["origin"] == "https://ekaterinburg.domclick.ru/pokupka/kvartiry/vtorichka" # ── fetch() cookies passthrough (DomClick cookie-injection, #2000) ─────────────── async def test_fetch_without_cookies_sends_none() -> None: """fetch(url) без cookies → payload содержит "cookies": None (parity avito/cian/yandex). Ни один провайдер кроме domclick debug-fetch не передаёт cookies — сервер (body.get("cookies")) трактует None как «инъекции не делать», поведение не меняется. """ client = _mock_client({"html": ""}) bf = _fetcher(client, source="avito") html = await bf.fetch("https://avito.ru/x") assert html == "" body = client.post.call_args.kwargs["json"] assert body["cookies"] is None async def test_fetch_with_cookies_sends_them_in_payload() -> None: """fetch(url, cookies=X) → payload несёт cookies=X (DomClick session cookie-injection).""" client = _mock_client({"html": ""}) bf = _fetcher(client, source="cian") session_cookies = {"CAS_ID": "12345", "qrator_jsid2": "abc"} await bf.fetch( "https://ekaterinburg.domclick.ru/card/sale__flat__1", cookies=session_cookies, ) body = client.post.call_args.kwargs["json"] assert body["cookies"] == session_cookies