gendesign/tradein-mvp/backend/tests/test_kit_browser_fetcher_proxy_pool.py

"""Тесты browser-пула для kit `scraper_kit.browser_fetcher.BrowserFetcher` (#2164 P4).

Инвариант ship-dark + fallback:
  - use_pool=False (дефолт) → в теле POST /fetch НЕТ поля "proxy"; proxy_provider не
    трогается (golden-parity: браузер юзает свой env-прокси BROWSER_PROXY_*).
  - use_pool=True + пул выдал lease → тело содержит "proxy"=lease.url + "proxy_kind";
    на выходе mark_health(ok) + release (в finally — lease не течёт).
  - use_pool=True + пул пуст (acquire→None) → тела без "proxy", НЕ падаем.
  - use_pool=True + acquire бросил → fallback без "proxy", НЕ падаем.
  - fetch кинул → mark_health(ok=False) + release всё равно (finally).

httpx полностью замокан: fetcher._client подменяется MagicMock'ом.
"""

from __future__ import annotations

from typing import Any
from unittest.mock import AsyncMock, MagicMock

import pytest
from scraper_kit.browser_fetcher import BrowserFetcher
from scraper_kit.contracts import ProxyLease


def _mock_client(json_payload: dict[str, Any], *, raise_exc: Exception | None = None) -> MagicMock:
    """httpx.AsyncClient-заглушка: .post → resp c raise_for_status/json."""
    resp = MagicMock()
    if raise_exc is not None:
        resp.raise_for_status.side_effect = raise_exc
    else:
        resp.raise_for_status.return_value = None
    resp.json.return_value = json_payload
    client = MagicMock()
    client.post = AsyncMock(return_value=resp)
    return client


class _FakeProxyProvider:
    """ProxyProvider-заглушка: acquire отдаёт заданный lease (или None), считает вызовы."""

    def __init__(self, lease: ProxyLease | None, *, acquire_raises: bool = False) -> None:
        self._lease = lease
        self._acquire_raises = acquire_raises
        self.acquired: list[str] = []
        self.released: list[int] = []
        self.health: list[tuple[int, bool]] = []

    def acquire(self, provider: str) -> ProxyLease | None:
        self.acquired.append(provider)
        if self._acquire_raises:
            raise RuntimeError("pool boom")
        return self._lease

    def release(self, lease: ProxyLease) -> None:
        self.released.append(lease.id)

    def mark_health(self, lease: ProxyLease, ok: bool, **_: Any) -> None:
        self.health.append((lease.id, ok))


def _fetcher(client: MagicMock, **kwargs: Any) -> BrowserFetcher:
    bf = BrowserFetcher(endpoint="http://browser:3000", **kwargs)
    bf._client = client
    return bf


async def test_fetch_pool_off_no_proxy_in_body() -> None:
    """use_pool=False (дефолт) → тело без 'proxy', provider не трогается (parity)."""
    provider = _FakeProxyProvider(ProxyLease(id=1, url="http://pool:8080", kind="http"))
    client = _mock_client({"html": "<ok>"})
    bf = _fetcher(client, source="avito", proxy_provider=provider)  # use_pool default False

    html = await bf.fetch("https://avito.ru/x")

    assert html == "<ok>"
    body = client.post.call_args.kwargs["json"]
    assert "proxy" not in body
    assert provider.acquired == []  # пул не трогали


async def test_fetch_pool_on_injects_proxy_and_releases() -> None:
    """use_pool=True + lease → 'proxy'/'proxy_kind' в теле; mark_health(True)+release."""
    lease = ProxyLease(id=7, url="http://u:p@pool:8080", kind="http")
    provider = _FakeProxyProvider(lease)
    client = _mock_client({"html": "<ok>"})
    bf = _fetcher(client, source="avito", proxy_provider=provider, use_pool=True)

    await bf.fetch("https://avito.ru/x")

    body = client.post.call_args.kwargs["json"]
    assert body["proxy"] == "http://u:p@pool:8080"
    assert body["proxy_kind"] == "http"
    assert provider.acquired == ["avito"]
    assert provider.health == [(7, True)]
    assert provider.released == [7]


async def test_fetch_pool_empty_falls_back_no_proxy() -> None:
    """use_pool=True + пул пуст (acquire→None) → тело без 'proxy', без mark_health/release."""
    provider = _FakeProxyProvider(None)
    client = _mock_client({"html": "<ok>"})
    bf = _fetcher(client, source="cian", proxy_provider=provider, use_pool=True)

    await bf.fetch("https://cian.ru/x")

    body = client.post.call_args.kwargs["json"]
    assert "proxy" not in body
    assert provider.acquired == ["cian"]
    assert provider.released == []
    assert provider.health == []


async def test_fetch_pool_acquire_error_falls_back() -> None:
    """acquire бросил → fallback без 'proxy', сбор НЕ падает."""
    provider = _FakeProxyProvider(None, acquire_raises=True)
    client = _mock_client({"html": "<ok>"})
    bf = _fetcher(client, source="yandex", proxy_provider=provider, use_pool=True)

    html = await bf.fetch("https://yandex.ru/x")

    assert html == "<ok>"
    body = client.post.call_args.kwargs["json"]
    assert "proxy" not in body
    assert provider.released == []


async def test_fetch_error_marks_health_false_and_releases() -> None:
    """fetch кинул → mark_health(ok=False) + release всё равно (finally, lease не течёт)."""
    lease = ProxyLease(id=3, url="http://pool:8080", kind="http")
    provider = _FakeProxyProvider(lease)
    # raise_for_status кидает НЕ-httpx ошибку → fetch() не ретраит, пробрасывает наверх.
    client = _mock_client({"html": "x"}, raise_exc=ValueError("bad status"))
    bf = _fetcher(client, source="avito", proxy_provider=provider, use_pool=True)

    with pytest.raises(ValueError):
        await bf.fetch("https://avito.ru/x")

    assert provider.health == [(3, False)]
    assert provider.released == [3]


async def test_fetch_json_pool_on_injects_proxy() -> None:
    """fetch_json тоже прокидывает proxy из пула в тело /fetch-json."""
    lease = ProxyLease(id=9, url="http://pool:8080", kind="http")
    provider = _FakeProxyProvider(lease)
    client = _mock_client({"status": 200, "body": "{}"})
    bf = _fetcher(client, source="avito", proxy_provider=provider, use_pool=True)

    await bf.fetch_json(
        "https://avito.ru/api", method="POST", body="{}", origin="https://avito.ru/"
    )

    body = client.post.call_args.kwargs["json"]
    assert body["proxy"] == "http://pool:8080"
    assert body["proxy_kind"] == "http"
    assert provider.released == [9]


async def test_fetch_json_pool_off_no_proxy() -> None:
    """use_pool=False → fetch_json без 'proxy' в теле (parity)."""
    client = _mock_client({"status": 200, "body": "{}"})
    bf = _fetcher(client, source="avito")

    await bf.fetch_json("https://avito.ru/api")

    body = client.post.call_args.kwargs["json"]
    assert "proxy" not in body


# ── fetch() origin passthrough (DomClick SERP-anchor, зеркалит fetch_json) ───────


async def test_fetch_without_origin_sends_none() -> None:
    """fetch(url) без origin → payload содержит "origin": None (parity avito/cian/yandex).

    Ни один провайдер кроме domclick не передаёт origin — сервер (body.get("origin"))
    трактует None как «origin-goto не выполнять», поведение не меняется.
    """
    client = _mock_client({"html": "<ok>"})
    bf = _fetcher(client, source="avito")

    html = await bf.fetch("https://avito.ru/x")

    assert html == "<ok>"
    body = client.post.call_args.kwargs["json"]
    assert body["origin"] is None


async def test_fetch_with_origin_sends_it_in_payload() -> None:
    """fetch(url, origin=X) → payload несёт origin=X (domclick vtorichka-SERP anchor)."""
    client = _mock_client({"html": "<ok>"})
    bf = _fetcher(client, source="cian")

    await bf.fetch(
        "https://ekaterinburg.domclick.ru/card/sale__flat__1",
        origin="https://ekaterinburg.domclick.ru/pokupka/kvartiry/vtorichka",
    )

    body = client.post.call_args.kwargs["json"]
    assert body["origin"] == "https://ekaterinburg.domclick.ru/pokupka/kvartiry/vtorichka"


# ── fetch() cookies passthrough (DomClick cookie-injection, #2000) ───────────────


async def test_fetch_without_cookies_sends_none() -> None:
    """fetch(url) без cookies → payload содержит "cookies": None (parity avito/cian/yandex).

    Ни один провайдер кроме domclick debug-fetch не передаёт cookies — сервер
    (body.get("cookies")) трактует None как «инъекции не делать», поведение не меняется.
    """
    client = _mock_client({"html": "<ok>"})
    bf = _fetcher(client, source="avito")

    html = await bf.fetch("https://avito.ru/x")

    assert html == "<ok>"
    body = client.post.call_args.kwargs["json"]
    assert body["cookies"] is None


async def test_fetch_with_cookies_sends_them_in_payload() -> None:
    """fetch(url, cookies=X) → payload несёт cookies=X (DomClick session cookie-injection)."""
    client = _mock_client({"html": "<ok>"})
    bf = _fetcher(client, source="cian")
    session_cookies = {"CAS_ID": "12345", "qrator_jsid2": "abc"}

    await bf.fetch(
        "https://ekaterinburg.domclick.ru/card/sale__flat__1",
        cookies=session_cookies,
    )

    body = client.post.call_args.kwargs["json"]
    assert body["cookies"] == session_cookies