"""Regression test — scraper_kit yandex/cian newbuilding providers proxy `endpoint=` into `BrowserFetcher` (issue #2322, epic #2277 Group F1). Bug: `scraper_kit/providers/yandex/newbuilding.py` (`YandexNewbuildingScraper.fetch_jk`, `resolve_yandex_jk_slug`) and `scraper_kit/providers/cian/newbuilding.py` (`fetch_newbuilding`) constructed `BrowserFetcher(source=...)` WITHOUT the mandatory `endpoint=` kwarg (required, no default in `BrowserFetcher.__init__`) — `TypeError` on every real call, and the functions did not even expose a `config` parameter so callers could not fix this from outside via DI (mirrors the pattern already used by `resolve_cian_zhk_url_via_search` / other providers, e.g. `cian/serp.py`, `domclick/serp.py`, `avito/serp.py`, `yandex/serp.py`). These tests prove `config.browser_http_endpoint` is actually threaded through to the `BrowserFetcher(...)` constructor call args — not merely that the function "doesn't crash" on a happy path. `BrowserFetcher` itself is replaced by a spy so the constructor call is directly inspectable. Also covers a SIBLING but importantly DIFFERENT footgun (#2397 Part D3, migrating `app.tasks.newbuilding_enrich_backfill` onto this kit provider): `resolve_cian_zhk_url_via_search` does NOT use `BrowserFetcher` at all — it goes through `build_curl_cffi_session` — so a missing `config=` does NOT raise like the `BrowserFetcher.__aenter__` assertion above. It silently degrades to a direct (non-proxied) connection (`cian_proxy_url` dropped). That is much harder to notice in prod (no crash, just quietly unproxied traffic that anti-bot can throttle/block), so a caller that forgets `config=` here gets no signal at all. """ from __future__ import annotations from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock import pytest from scraper_kit.providers.cian.newbuilding import ( fetch_newbuilding, resolve_cian_zhk_url_via_search, ) from scraper_kit.providers.yandex.newbuilding import ( YandexNewbuildingScraper, resolve_yandex_jk_slug, ) _TEST_ENDPOINT = "http://test-tradein-browser:9009/fetch" _TEST_PROXY = "http://mobile-proxy.example:8080" def _spy_browser_fetcher(html: str) -> MagicMock: """BrowserFetcher class-replacement spy: records the constructor call args/kwargs and returns an async-context-manager mock whose `.fetch()` resolves to `html`.""" mock_fetcher = MagicMock() mock_fetcher.fetch = AsyncMock(return_value=html) mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher) mock_fetcher.__aexit__ = AsyncMock(return_value=None) return MagicMock(return_value=mock_fetcher) # ── cian.newbuilding.fetch_newbuilding ─────────────────────────────────────── @pytest.mark.asyncio async def test_cian_fetch_newbuilding_passes_endpoint_from_config(monkeypatch): """config.browser_http_endpoint должен попасть в BrowserFetcher(endpoint=...).""" spy = _spy_browser_fetcher("") monkeypatch.setattr("scraper_kit.providers.cian.newbuilding.BrowserFetcher", spy) config = SimpleNamespace(browser_http_endpoint=_TEST_ENDPOINT) await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", config=config) spy.assert_called_once() _, kwargs = spy.call_args assert kwargs.get("source") == "cian" assert kwargs.get("endpoint") == _TEST_ENDPOINT @pytest.mark.asyncio async def test_cian_fetch_newbuilding_endpoint_none_without_config(monkeypatch): """Без config= (backward-compat) endpoint=None передаётся явным kwarg'ом — сигнатура больше не роняет TypeError на отсутствующем обязательном параметре конструктора.""" spy = _spy_browser_fetcher("") monkeypatch.setattr("scraper_kit.providers.cian.newbuilding.BrowserFetcher", spy) await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/") spy.assert_called_once() _, kwargs = spy.call_args assert kwargs.get("source") == "cian" assert kwargs.get("endpoint") is None # ── yandex.newbuilding.YandexNewbuildingScraper.fetch_jk ───────────────────── @pytest.mark.asyncio async def test_yandex_fetch_jk_passes_endpoint_from_config(monkeypatch): """config передан в конструктор скрапера → endpoint пробрасывается в BrowserFetcher.""" spy = _spy_browser_fetcher("") monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy) config = SimpleNamespace(browser_http_endpoint=_TEST_ENDPOINT) scraper = YandexNewbuildingScraper(config=config) await scraper.fetch_jk(jk_slug="tatlin", jk_id="1592987") spy.assert_called_once() _, kwargs = spy.call_args assert kwargs.get("source") == "yandex" assert kwargs.get("endpoint") == _TEST_ENDPOINT @pytest.mark.asyncio async def test_yandex_fetch_jk_endpoint_none_without_config(monkeypatch): """Без config= (backward-compat) endpoint=None передаётся явным kwarg'ом.""" spy = _spy_browser_fetcher("") monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy) scraper = YandexNewbuildingScraper() await scraper.fetch_jk(jk_slug="tatlin", jk_id="1592987") spy.assert_called_once() _, kwargs = spy.call_args assert kwargs.get("source") == "yandex" assert kwargs.get("endpoint") is None # ── yandex.newbuilding.resolve_yandex_jk_slug ──────────────────────────────── @pytest.mark.asyncio async def test_resolve_yandex_jk_slug_passes_endpoint_from_config(monkeypatch): """config= (новый kwarg-only параметр) должен пробрасывать endpoint в BrowserFetcher.""" spy = _spy_browser_fetcher("") monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy) config = SimpleNamespace(browser_http_endpoint=_TEST_ENDPOINT) await resolve_yandex_jk_slug("1592987", config=config) spy.assert_called_once() _, kwargs = spy.call_args assert kwargs.get("source") == "yandex" assert kwargs.get("endpoint") == _TEST_ENDPOINT @pytest.mark.asyncio async def test_resolve_yandex_jk_slug_endpoint_none_without_config(monkeypatch): """Без config= (backward-compat) endpoint=None передаётся явным kwarg'ом.""" spy = _spy_browser_fetcher("") monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy) await resolve_yandex_jk_slug("1592987") spy.assert_called_once() _, kwargs = spy.call_args assert kwargs.get("source") == "yandex" assert kwargs.get("endpoint") is None # ── cian.newbuilding.resolve_cian_zhk_url_via_search ───────────────────────── # NOTE (#2397 Part D3): this provider does NOT use BrowserFetcher (unlike # fetch_newbuilding above) — it builds its own curl_cffi session via # build_curl_cffi_session(proxy_url=...). Unlike the BrowserFetcher assertion, a # missing config= here does NOT crash — it silently drops the proxy and falls back to # a direct (non-proxied) connection. These tests prove the threading in BOTH directions # so a caller forgetting config= has a red test instead of quiet unproxied traffic. _SERP_HTML = ( '

Купить квартиру в ' 'ЖК «Парковый квартал»

' ) def _spy_curl_session(html: str) -> MagicMock: """build_curl_cffi_session spy: records the (proxy_url=...) call kwargs and returns a fake curl_cffi AsyncSession whose .get() resolves to a 200 response carrying `html`.""" mock_resp = SimpleNamespace(status_code=200, text=html) mock_session = MagicMock() mock_session.get = AsyncMock(return_value=mock_resp) mock_session.close = AsyncMock() return MagicMock(return_value=mock_session) @pytest.mark.asyncio async def test_resolve_cian_zhk_url_via_search_passes_proxy_from_config(monkeypatch): """config.cian_proxy_url должен попасть в build_curl_cffi_session(proxy_url=...).""" spy = _spy_curl_session(_SERP_HTML) monkeypatch.setattr("scraper_kit.providers.cian.newbuilding.build_curl_cffi_session", spy) config = SimpleNamespace(cian_proxy_url=_TEST_PROXY) url = await resolve_cian_zhk_url_via_search(48853, config=config) assert url == "https://zhk-parkovyy-kvartal-ekb-i.cian.ru" spy.assert_called_once() _, kwargs = spy.call_args assert kwargs.get("proxy_url") == _TEST_PROXY @pytest.mark.asyncio async def test_resolve_cian_zhk_url_via_search_proxy_none_without_config(monkeypatch): """Без config= (SILENT footgun): вызов НЕ падает (в отличие от BrowserFetcher assertion выше в этом файле), но proxy_url молча становится None — резолвер уходит на прямое (непроксированное) соединение вместо настроенного мобильного прокси.""" spy = _spy_curl_session(_SERP_HTML) monkeypatch.setattr("scraper_kit.providers.cian.newbuilding.build_curl_cffi_session", spy) url = await resolve_cian_zhk_url_via_search(48853) assert url == "https://zhk-parkovyy-kvartal-ekb-i.cian.ru" # "works" — silently unproxied spy.assert_called_once() _, kwargs = spy.call_args assert kwargs.get("proxy_url") is None