"""Тесты shared-фундамента `scraper_kit.providers._base` (#2358, Foundation). Покрывает: - `http_proxies()` — url → dict / None passthrough. - `build_curl_cffi_session()` / `build_document_session()` — корректные kwargs в curl_cffi `AsyncSession` (impersonate/timeout/proxies/headers/cookies). - `build_browser_fetcher()` — `config` MANDATORY (нет default в сигнатуре); endpoint/use_pool/proxy_provider корректно прокинуты; `fetch_timeout_s` опционален (дефолт `BrowserFetcher`, если не передан явно). Изолированные — никакой интеграции с existing providers/avito|cian|yandex|domclick. """ from __future__ import annotations import inspect from typing import Any from scraper_kit.browser_fetcher import BrowserFetcher from scraper_kit.providers._base import ( DEFAULT_IMPERSONATE, DOCUMENT_HEADERS, build_browser_fetcher, build_curl_cffi_session, build_document_session, http_proxies, ) class _FakeConfig: """Минимальная ScraperConfig-заглушка — только поля, читаемые build_browser_fetcher.""" def __init__(self, endpoint: str, use_pool: bool = False) -> None: self.browser_http_endpoint = endpoint self.use_proxy_pool_browser = use_pool class _FakeProxyProvider: """ProxyProvider-заглушка — сюда достаточно структурного соответствия, вызовы не нужны.""" def acquire(self, provider: str) -> Any: raise NotImplementedError def release(self, lease: Any) -> None: raise NotImplementedError def mark_health(self, lease: Any, ok: bool, **_: Any) -> None: raise NotImplementedError # ── http_proxies ────────────────────────────────────────────────────────────── def test_http_proxies_none_passthrough() -> None: assert http_proxies(None) is None def test_http_proxies_empty_string_is_none() -> None: assert http_proxies("") is None def test_http_proxies_builds_dict_for_both_schemes() -> None: result = http_proxies("http://user:pass@proxy:8080") assert result == { "http": "http://user:pass@proxy:8080", "https": "http://user:pass@proxy:8080", } # ── build_curl_cffi_session / build_document_session ────────────────────────── async def test_build_curl_cffi_session_wires_impersonate_timeout_proxies() -> None: session = build_curl_cffi_session(proxy_url="http://proxy:8080", timeout=15.0) try: assert session.impersonate == DEFAULT_IMPERSONATE assert session.timeout == 15.0 assert session.proxies == {"http": "http://proxy:8080", "https": "http://proxy:8080"} finally: await session.close() async def test_build_curl_cffi_session_no_proxy_direct_connection() -> None: session = build_curl_cffi_session(proxy_url=None) try: # curl_cffi нормализует proxies=None в {} (falsy) — прямое подключение. assert not session.proxies finally: await session.close() async def test_build_curl_cffi_session_passes_headers_and_cookies() -> None: session = build_curl_cffi_session( proxy_url=None, headers={"X-Test": "1"}, cookies={"sid": "abc"} ) try: assert dict(session.headers).get("x-test") == "1" assert session.cookies.get("sid") == "abc" finally: await session.close() async def test_build_document_session_uses_document_headers() -> None: session = build_document_session(proxy_url=None) try: headers = dict(session.headers) for key, value in DOCUMENT_HEADERS.items(): assert headers.get(key.lower()) == value finally: await session.close() async def test_build_document_session_custom_timeout_and_proxy() -> None: session = build_document_session(proxy_url="http://proxy:8080", timeout=10.0) try: assert session.timeout == 10.0 assert session.proxies == {"http": "http://proxy:8080", "https": "http://proxy:8080"} finally: await session.close() # ── build_browser_fetcher ────────────────────────────────────────────────────── def test_build_browser_fetcher_config_is_mandatory_in_signature() -> None: """config — параметр БЕЗ default в сигнатуре: пропустить его невозможно (footgun-guard).""" sig = inspect.signature(build_browser_fetcher) config_param = sig.parameters["config"] assert config_param.default is inspect.Parameter.empty def test_build_browser_fetcher_wires_endpoint_and_use_pool() -> None: config = _FakeConfig(endpoint="http://browser:3000", use_pool=True) provider = _FakeProxyProvider() fetcher = build_browser_fetcher(config, "avito", proxy_provider=provider) # type: ignore[arg-type] assert isinstance(fetcher, BrowserFetcher) assert fetcher._endpoint == "http://browser:3000" assert fetcher._use_pool is True assert fetcher._proxy_provider is provider assert fetcher._source == "avito" def test_build_browser_fetcher_use_pool_false_by_default_config() -> None: config = _FakeConfig(endpoint="http://browser:3000", use_pool=False) fetcher = build_browser_fetcher(config, "cian") # type: ignore[arg-type] assert fetcher._use_pool is False assert fetcher._proxy_provider is None def test_build_browser_fetcher_default_timeout_when_not_specified() -> None: config = _FakeConfig(endpoint="http://browser:3000") fetcher = build_browser_fetcher(config, "yandex") # type: ignore[arg-type] # Дефолт BrowserFetcher (_HTTP_TIMEOUT_S = 120.0) — не переопределён. assert fetcher._fetch_timeout_s == 120.0 def test_build_browser_fetcher_explicit_timeout_override() -> None: config = _FakeConfig(endpoint="http://browser:3000") fetcher = build_browser_fetcher(config, "yandex", fetch_timeout_s=30.0) # type: ignore[arg-type] assert fetcher._fetch_timeout_s == 30.0