5015109f31
All checks were successful
CI Trade-In / changes (pull_request) Successful in 8s
CI / changes (pull_request) Successful in 7s
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
CI Trade-In / backend-tests (pull_request) Successful in 1m2s
208 lines
9.5 KiB
Python
208 lines
9.5 KiB
Python
"""Regression test — scraper_kit yandex/cian newbuilding providers proxy `endpoint=`
|
||
into `BrowserFetcher` (issue #2322, epic #2277 Group F1).
|
||
|
||
Bug: `scraper_kit/providers/yandex/newbuilding.py` (`YandexNewbuildingScraper.fetch_jk`,
|
||
`resolve_yandex_jk_slug`) and `scraper_kit/providers/cian/newbuilding.py`
|
||
(`fetch_newbuilding`) constructed `BrowserFetcher(source=...)` WITHOUT the mandatory
|
||
`endpoint=` kwarg (required, no default in `BrowserFetcher.__init__`) — `TypeError` on
|
||
every real call, and the functions did not even expose a `config` parameter so callers
|
||
could not fix this from outside via DI (mirrors the pattern already used by
|
||
`resolve_cian_zhk_url_via_search` / other providers, e.g. `cian/serp.py`,
|
||
`domclick/serp.py`, `avito/serp.py`, `yandex/serp.py`).
|
||
|
||
These tests prove `config.browser_http_endpoint` is actually threaded through to the
|
||
`BrowserFetcher(...)` constructor call args — not merely that the function "doesn't
|
||
crash" on a happy path. `BrowserFetcher` itself is replaced by a spy so the constructor
|
||
call is directly inspectable.
|
||
|
||
Also covers a SIBLING but importantly DIFFERENT footgun (#2397 Part D3, migrating
|
||
`app.tasks.newbuilding_enrich_backfill` onto this kit provider):
|
||
`resolve_cian_zhk_url_via_search` does NOT use `BrowserFetcher` at all — it goes
|
||
through `build_curl_cffi_session` — so a missing `config=` does NOT raise like the
|
||
`BrowserFetcher.__aenter__` assertion above. It silently degrades to a direct
|
||
(non-proxied) connection (`cian_proxy_url` dropped). That is much harder to notice in
|
||
prod (no crash, just quietly unproxied traffic that anti-bot can throttle/block), so a
|
||
caller that forgets `config=` here gets no signal at all.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from types import SimpleNamespace
|
||
from unittest.mock import AsyncMock, MagicMock
|
||
|
||
import pytest
|
||
from scraper_kit.providers.cian.newbuilding import (
|
||
fetch_newbuilding,
|
||
resolve_cian_zhk_url_via_search,
|
||
)
|
||
from scraper_kit.providers.yandex.newbuilding import (
|
||
YandexNewbuildingScraper,
|
||
resolve_yandex_jk_slug,
|
||
)
|
||
|
||
_TEST_ENDPOINT = "http://test-tradein-browser:9009/fetch"
|
||
_TEST_PROXY = "http://mobile-proxy.example:8080"
|
||
|
||
|
||
def _spy_browser_fetcher(html: str) -> MagicMock:
|
||
"""BrowserFetcher class-replacement spy: records the constructor call args/kwargs
|
||
and returns an async-context-manager mock whose `.fetch()` resolves to `html`."""
|
||
mock_fetcher = MagicMock()
|
||
mock_fetcher.fetch = AsyncMock(return_value=html)
|
||
mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
|
||
mock_fetcher.__aexit__ = AsyncMock(return_value=None)
|
||
return MagicMock(return_value=mock_fetcher)
|
||
|
||
|
||
# ── cian.newbuilding.fetch_newbuilding ───────────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_cian_fetch_newbuilding_passes_endpoint_from_config(monkeypatch):
|
||
"""config.browser_http_endpoint должен попасть в BrowserFetcher(endpoint=...)."""
|
||
spy = _spy_browser_fetcher("<html></html>")
|
||
monkeypatch.setattr("scraper_kit.providers.cian.newbuilding.BrowserFetcher", spy)
|
||
|
||
config = SimpleNamespace(browser_http_endpoint=_TEST_ENDPOINT)
|
||
await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", config=config)
|
||
|
||
spy.assert_called_once()
|
||
_, kwargs = spy.call_args
|
||
assert kwargs.get("source") == "cian"
|
||
assert kwargs.get("endpoint") == _TEST_ENDPOINT
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_cian_fetch_newbuilding_endpoint_none_without_config(monkeypatch):
|
||
"""Без config= (backward-compat) endpoint=None передаётся явным kwarg'ом — сигнатура
|
||
больше не роняет TypeError на отсутствующем обязательном параметре конструктора."""
|
||
spy = _spy_browser_fetcher("<html></html>")
|
||
monkeypatch.setattr("scraper_kit.providers.cian.newbuilding.BrowserFetcher", spy)
|
||
|
||
await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/")
|
||
|
||
spy.assert_called_once()
|
||
_, kwargs = spy.call_args
|
||
assert kwargs.get("source") == "cian"
|
||
assert kwargs.get("endpoint") is None
|
||
|
||
|
||
# ── yandex.newbuilding.YandexNewbuildingScraper.fetch_jk ─────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_yandex_fetch_jk_passes_endpoint_from_config(monkeypatch):
|
||
"""config передан в конструктор скрапера → endpoint пробрасывается в BrowserFetcher."""
|
||
spy = _spy_browser_fetcher("<html></html>")
|
||
monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy)
|
||
|
||
config = SimpleNamespace(browser_http_endpoint=_TEST_ENDPOINT)
|
||
scraper = YandexNewbuildingScraper(config=config)
|
||
await scraper.fetch_jk(jk_slug="tatlin", jk_id="1592987")
|
||
|
||
spy.assert_called_once()
|
||
_, kwargs = spy.call_args
|
||
assert kwargs.get("source") == "yandex"
|
||
assert kwargs.get("endpoint") == _TEST_ENDPOINT
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_yandex_fetch_jk_endpoint_none_without_config(monkeypatch):
|
||
"""Без config= (backward-compat) endpoint=None передаётся явным kwarg'ом."""
|
||
spy = _spy_browser_fetcher("<html></html>")
|
||
monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy)
|
||
|
||
scraper = YandexNewbuildingScraper()
|
||
await scraper.fetch_jk(jk_slug="tatlin", jk_id="1592987")
|
||
|
||
spy.assert_called_once()
|
||
_, kwargs = spy.call_args
|
||
assert kwargs.get("source") == "yandex"
|
||
assert kwargs.get("endpoint") is None
|
||
|
||
|
||
# ── yandex.newbuilding.resolve_yandex_jk_slug ────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_resolve_yandex_jk_slug_passes_endpoint_from_config(monkeypatch):
|
||
"""config= (новый kwarg-only параметр) должен пробрасывать endpoint в BrowserFetcher."""
|
||
spy = _spy_browser_fetcher("<html></html>")
|
||
monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy)
|
||
|
||
config = SimpleNamespace(browser_http_endpoint=_TEST_ENDPOINT)
|
||
await resolve_yandex_jk_slug("1592987", config=config)
|
||
|
||
spy.assert_called_once()
|
||
_, kwargs = spy.call_args
|
||
assert kwargs.get("source") == "yandex"
|
||
assert kwargs.get("endpoint") == _TEST_ENDPOINT
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_resolve_yandex_jk_slug_endpoint_none_without_config(monkeypatch):
|
||
"""Без config= (backward-compat) endpoint=None передаётся явным kwarg'ом."""
|
||
spy = _spy_browser_fetcher("<html></html>")
|
||
monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy)
|
||
|
||
await resolve_yandex_jk_slug("1592987")
|
||
|
||
spy.assert_called_once()
|
||
_, kwargs = spy.call_args
|
||
assert kwargs.get("source") == "yandex"
|
||
assert kwargs.get("endpoint") is None
|
||
|
||
|
||
# ── cian.newbuilding.resolve_cian_zhk_url_via_search ─────────────────────────
|
||
# NOTE (#2397 Part D3): this provider does NOT use BrowserFetcher (unlike
|
||
# fetch_newbuilding above) — it builds its own curl_cffi session via
|
||
# build_curl_cffi_session(proxy_url=...). Unlike the BrowserFetcher assertion, a
|
||
# missing config= here does NOT crash — it silently drops the proxy and falls back to
|
||
# a direct (non-proxied) connection. These tests prove the threading in BOTH directions
|
||
# so a caller forgetting config= has a red test instead of quiet unproxied traffic.
|
||
|
||
_SERP_HTML = (
|
||
'<h1 data-name="Title">Купить квартиру в '
|
||
'<a href="https://zhk-parkovyy-kvartal-ekb-i.cian.ru/">ЖК «Парковый квартал»</a></h1>'
|
||
)
|
||
|
||
|
||
def _spy_curl_session(html: str) -> MagicMock:
|
||
"""build_curl_cffi_session spy: records the (proxy_url=...) call kwargs and returns
|
||
a fake curl_cffi AsyncSession whose .get() resolves to a 200 response carrying `html`."""
|
||
mock_resp = SimpleNamespace(status_code=200, text=html)
|
||
mock_session = MagicMock()
|
||
mock_session.get = AsyncMock(return_value=mock_resp)
|
||
mock_session.close = AsyncMock()
|
||
return MagicMock(return_value=mock_session)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_resolve_cian_zhk_url_via_search_passes_proxy_from_config(monkeypatch):
|
||
"""config.cian_proxy_url должен попасть в build_curl_cffi_session(proxy_url=...)."""
|
||
spy = _spy_curl_session(_SERP_HTML)
|
||
monkeypatch.setattr("scraper_kit.providers.cian.newbuilding.build_curl_cffi_session", spy)
|
||
|
||
config = SimpleNamespace(cian_proxy_url=_TEST_PROXY)
|
||
url = await resolve_cian_zhk_url_via_search(48853, config=config)
|
||
|
||
assert url == "https://zhk-parkovyy-kvartal-ekb-i.cian.ru"
|
||
spy.assert_called_once()
|
||
_, kwargs = spy.call_args
|
||
assert kwargs.get("proxy_url") == _TEST_PROXY
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_resolve_cian_zhk_url_via_search_proxy_none_without_config(monkeypatch):
|
||
"""Без config= (SILENT footgun): вызов НЕ падает (в отличие от BrowserFetcher
|
||
assertion выше в этом файле), но proxy_url молча становится None — резолвер уходит
|
||
на прямое (непроксированное) соединение вместо настроенного мобильного прокси."""
|
||
spy = _spy_curl_session(_SERP_HTML)
|
||
monkeypatch.setattr("scraper_kit.providers.cian.newbuilding.build_curl_cffi_session", spy)
|
||
|
||
url = await resolve_cian_zhk_url_via_search(48853)
|
||
|
||
assert url == "https://zhk-parkovyy-kvartal-ekb-i.cian.ru" # "works" — silently unproxied
|
||
spy.assert_called_once()
|
||
_, kwargs = spy.call_args
|
||
assert kwargs.get("proxy_url") is None
|